forked from security_vision/semantic_graph
Compare commits
3 Commits
815bb200be
...
4ba47c9234
Author | SHA1 | Date |
---|---|---|
Ruben van de Ven | 4ba47c9234 | |
Ruben van de Ven | fb01595957 | |
Ruben van de Ven | 4bf9cfd954 |
49
README.md
49
README.md
|
@ -4,54 +4,21 @@ This repository contains a script to pull semantic data out of Semantic Mediawik
|
|||
|
||||
Built for [Security Vision](https://securityvision.io).
|
||||
|
||||
## Document
|
||||
|
||||
```
|
||||
pandoc EU\ Greens\ Report\ -\ Biometric\ Mass\ Surveillance\ v.3.docx --extract-media=images --template=template.html --section-divs -s -o output.html --metadata title="Biometric and Behavioural Mass Surveillance in EU Member States" -c mvp.css --toc --toc-depth=2
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
wget https://d3js.org/d3.v6.min.js
|
||||
```
|
||||
|
||||
## Update:
|
||||
|
||||
```
|
||||
python wiki_relations.py
|
||||
```
|
||||
|
||||
## Data
|
||||
|
||||
_Ask_ SMW with the following query:
|
||||
|
||||
```
|
||||
({{#ask: [[Category:Deployments||Institution]]
|
||||
OR [[Category:Technologies]] [[Developed by (institutions)::+]] // TODO: + should give a subquery <q></q> with all institutions in EU
|
||||
OR [[Category:Technologies]] [[-Software Deployed::+]] // TODO: + should give a subquery <q></q> with all deployments in EU
|
||||
|?Category
|
||||
|?Geolocation
|
||||
|?City
|
||||
|?City.Has Coordinates=Has Coordinates
|
||||
|?City.Is in Country=City Country
|
||||
|?City Country.Has Coordinates=Country Coordinates
|
||||
|?Clients
|
||||
|?Managed by
|
||||
|?Used by
|
||||
|?Funded by
|
||||
|?Provided by // TODO: technology/product provided by
|
||||
|?Software Deployed
|
||||
|?Developped by (institutions)
|
||||
|format=broadtable
|
||||
|limit=500
|
||||
|offset=0
|
||||
|link=all
|
||||
|sort=
|
||||
|order=asc
|
||||
|headers=show
|
||||
|searchlabel=... further results
|
||||
|class=sortable wikitable smwtable
|
||||
}}
|
||||
|
||||
TODO: see: https://www.semantic-mediawiki.org/wiki/Help:Querying_for_the_absence_of_a_property
|
||||
|
||||
{{#ask: [[Category:Deployments||Institution]] [[Is in Greens Report 2021::True]]
|
||||
|?Category
|
||||
|?Geolocation
|
||||
|
@ -81,10 +48,4 @@ TODO: see: https://www.semantic-mediawiki.org/wiki/Help:Querying_for_the_absence
|
|||
|searchlabel=... further results
|
||||
|class=sortable wikitable smwtable
|
||||
}}
|
||||
```
|
||||
|
||||
Fetch using CURL:
|
||||
|
||||
```bash
|
||||
curl 'https://www.securityvision.io/wiki/index.php?title=Special:Ask&#search' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: application/x-www-form-urlencoded' -H 'Origin: https://www.securityvision.io' -H 'Connection: keep-alive' -H 'Referer: https://www.securityvision.io/wiki/index.php/Special:Ask' -H 'Upgrade-Insecure-Requests: 1' --data-raw 'title=Special%3AAsk&_action=submit&q=%5B%5BCategory%3ADeployments%7C%7CInstitution%5D%5D++&po=%0D%0A+%7C%3FCategory%0D%0A+%7C%3FGeolocation%0D%0A+%7C%3FCity%0D%0A+%7C%3FCity.Has+Coordinates%3DCity+Coordinates%0D%0A+%7C%3FCity.Is+in+Country%3DCity+Country%0D%0A+%7C%3FCity+Country.Has+Coordinates%3DCountry+Coordinates%0D%0A+%7C%3FClients%0D%0A+%7C%3FManaged+by%0D%0A+%7C%3FUsed+by%0D%0A+%7C%3FFunded+by%0D%0A+%7C%3FProvided+by%0D%0A+%7C%3FSoftware+Deployed%0D%0A+%7C%3FSoftware+Deployed.Developped+by+%28institutions%29%3DSoftware+Developer%0D%0A+%7C%3FDatasets+Used%0D%0A+%7C%3FDatasets+Used.Developed+by+Institution%3DDataset+Developer&eq=yes&p%5Bformat%5D=json&p%5Blimit%5D=500&p%5Boffset%5D=0&p%5Blink%5D=all&p%5Bheaders%5D=show&p%5Bmainlabel%5D=&p%5Bintro%5D=&p%5Boutro%5D=&p%5Bsearchlabel%5D=JSON&p%5Bdefault%5D=&p%5Btype%5D=full&p%5Bfilename%5D=result2.json&sort_num%5B%5D=&order_num%5B%5D=asc&sort_num%5B%5D=&order_num%5B%5D=asc&eq=yes'
|
||||
```
|
Binary file not shown.
After Width: | Height: | Size: 207 KiB |
Binary file not shown.
After Width: | Height: | Size: 1.1 MiB |
Binary file not shown.
After Width: | Height: | Size: 4.0 MiB |
Binary file not shown.
After Width: | Height: | Size: 70 KiB |
Binary file not shown.
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
After Width: | Height: | Size: 16 KiB |
|
@ -0,0 +1,568 @@
|
|||
/* MVP.css v1.7.3 - https://github.com/andybrewer/mvp */
|
||||
|
||||
:root {
|
||||
--border-radius: 5px;
|
||||
--box-shadow: 2px 2px 10px;
|
||||
--color: darkblue;
|
||||
--hover-color: blue;
|
||||
--color-accent: #118bee15;
|
||||
--color-bg: #fff;
|
||||
--color-bg-secondary: #e9e9e9;
|
||||
--color-secondary: darkblue; /*#920de9;*/
|
||||
--color-secondary-accent: #920de90b;
|
||||
--color-shadow: #f4f4f4;
|
||||
--color-text: #000;
|
||||
--color-text-secondary: #999;
|
||||
--font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
|
||||
--hover-brightness: 1.2;
|
||||
--justify-important: center;
|
||||
--justify-normal: left;
|
||||
--line-height: 1.5;
|
||||
--width-card: 285px;
|
||||
--width-card-medium: 460px;
|
||||
--width-card-wide: 800px;
|
||||
--width-content: 720px;
|
||||
}
|
||||
|
||||
/*
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--color: #0097fc;
|
||||
--color-accent: #0097fc4f;
|
||||
--color-bg: #333;
|
||||
--color-bg-secondary: #555;
|
||||
--color-secondary: #e20de9;
|
||||
--color-secondary-accent: #e20de94f;
|
||||
--color-shadow: #bbbbbb20;
|
||||
--color-text: #f7f7f7;
|
||||
--color-text-secondary: #aaa;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
/* Layout */
|
||||
article aside {
|
||||
background: var(--color-secondary-accent);
|
||||
border-left: 4px solid var(--color-secondary);
|
||||
padding: 0.01rem 0.8rem;
|
||||
}
|
||||
|
||||
body {
|
||||
background: var(--color-bg);
|
||||
color: var(--color-text);
|
||||
font-family: var(--font-family);
|
||||
line-height: var(--line-height);
|
||||
margin: 0;
|
||||
overflow-x: hidden;
|
||||
padding: 1rem 0;
|
||||
}
|
||||
|
||||
footer,
|
||||
header,
|
||||
main {
|
||||
margin: 0 auto;
|
||||
max-width: var(--width-content);
|
||||
width: calc(100% - 2rem);
|
||||
padding: .5rem 1rem;
|
||||
}
|
||||
|
||||
header{
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: calc(50% - var(--width-content) / 2);
|
||||
padding-left: 0;
|
||||
padding-right: 0;
|
||||
background-color: white;
|
||||
border-bottom: solid 1px var(--color-bg-secondary);
|
||||
z-index: 999;
|
||||
}
|
||||
|
||||
@media screen and (max-width: 740px) {
|
||||
header{
|
||||
left: 0;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
hr {
|
||||
background-color: var(--color-bg-secondary);
|
||||
border: none;
|
||||
height: 1px;
|
||||
margin: 4rem 0;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
section {
|
||||
justify-content: var(--justify-important);
|
||||
}
|
||||
|
||||
section img,
|
||||
article img {
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
section pre {
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
section aside {
|
||||
border: 1px solid var(--color-bg-secondary);
|
||||
border-radius: var(--border-radius);
|
||||
box-shadow: var(--box-shadow) var(--color-shadow);
|
||||
margin: 1rem;
|
||||
padding: 1.25rem;
|
||||
width: var(--width-card);
|
||||
}
|
||||
|
||||
section aside:hover {
|
||||
box-shadow: var(--box-shadow) var(--color-bg-secondary);
|
||||
}
|
||||
|
||||
[hidden] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Headers */
|
||||
article header,
|
||||
div header,
|
||||
main header {
|
||||
padding-top: 0;
|
||||
}
|
||||
|
||||
header {
|
||||
/* text-align: var(--justify-important); */
|
||||
}
|
||||
|
||||
header a b,
|
||||
header a em,
|
||||
header a i,
|
||||
header a strong {
|
||||
margin-left: 0.5rem;
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
|
||||
|
||||
header nav > a{
|
||||
color: var(--color-text);
|
||||
padding-left: .5rem
|
||||
}
|
||||
header nav img {
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
section header {
|
||||
padding-top: 0;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
/* Nav */
|
||||
nav {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
font-weight: bold;
|
||||
justify-content: space-between;
|
||||
/* margin-bottom: 7rem; */
|
||||
}
|
||||
|
||||
nav ul {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
nav ul li {
|
||||
display: inline-block;
|
||||
margin: 0 0.5rem;
|
||||
position: relative;
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
/* Nav Dropdown */
|
||||
nav ul li:hover ul {
|
||||
display: block;
|
||||
}
|
||||
|
||||
nav ul li ul {
|
||||
background: var(--color-bg);
|
||||
border: 1px solid var(--color-bg-secondary);
|
||||
border-radius: var(--border-radius);
|
||||
box-shadow: var(--box-shadow) var(--color-shadow);
|
||||
display: none;
|
||||
height: auto;
|
||||
right: -2px;
|
||||
padding: .5rem 1rem;
|
||||
position: absolute;
|
||||
top: 1.7rem;
|
||||
white-space: nowrap;
|
||||
width: auto;
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
nav ul li ul li.space{
|
||||
margin-top:1rem;
|
||||
}
|
||||
|
||||
nav ul li ul::before {
|
||||
/* fill gap above to make mousing over them easier */
|
||||
content: "";
|
||||
position: absolute;
|
||||
left: 0;
|
||||
right: 0;
|
||||
top: -0.5rem;
|
||||
height: 0.5rem;
|
||||
}
|
||||
|
||||
nav ul li ul li,
|
||||
nav ul li ul li a {
|
||||
display: block;
|
||||
}
|
||||
|
||||
/* Typography */
|
||||
code,
|
||||
samp {
|
||||
background-color: var(--color-accent);
|
||||
border-radius: var(--border-radius);
|
||||
color: var(--color-text);
|
||||
display: inline-block;
|
||||
margin: 0 0.1rem;
|
||||
padding: 0 0.5rem;
|
||||
}
|
||||
|
||||
details {
|
||||
margin: 1.3rem 0;
|
||||
}
|
||||
|
||||
details summary {
|
||||
font-weight: bold;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
h1,
|
||||
h2,
|
||||
h3,
|
||||
h4,
|
||||
h5,
|
||||
h6 {
|
||||
text-align: left;
|
||||
line-height: var(--line-height);
|
||||
padding-top:5rem
|
||||
}
|
||||
|
||||
h1{
|
||||
padding-top: 100px;
|
||||
margin-bottom: 4rem;
|
||||
font-size: 150%;
|
||||
}
|
||||
h2{
|
||||
font-size: 125%;
|
||||
}
|
||||
/* h1::before{
|
||||
content: "CHAPTER " counter(h1counter) ".\0000a0\0000a0";
|
||||
counter-increment: h1counter;
|
||||
counter-reset: h2counter;
|
||||
}
|
||||
h2:before {
|
||||
content: counter(h1counter) "." counter(h2counter) ".\0000a0\0000a0";
|
||||
counter-increment: h2counter;
|
||||
counter-reset: h3counter;
|
||||
}
|
||||
h3:before {
|
||||
content: counter(h1counter) "." counter(h2counter) "." counter(h3counter) ".\0000a0\0000a0";
|
||||
counter-increment: h3counter;
|
||||
} */
|
||||
|
||||
/* thanks: http://philarcher.org/diary/2013/headingnumbers/ */
|
||||
body {counter-reset: h1}
|
||||
h1 {counter-reset: h2}
|
||||
h2 {counter-reset: h3}
|
||||
h3 {counter-reset: h4}
|
||||
h4 {counter-reset: h5}
|
||||
h5 {counter-reset: h6}
|
||||
|
||||
h1:before {counter-increment: h1; content: "CHAPTER "counter(h1) ". " }
|
||||
h2:before {counter-increment: h2; content: counter(h1) "." counter(h2) ". "}
|
||||
h3:before {counter-increment: h3; content: counter(h1) "." counter(h2) "." counter(h3) ". "}
|
||||
|
||||
h1.nocount:before, h1.title:before, h1.Title:before, h2.nocount:before, h3.nocount:before, h4.nocount:before, h5.nocount:before, h6.nocount:before { content: ""; counter-increment: none }
|
||||
|
||||
|
||||
mark {
|
||||
padding: 0.1rem;
|
||||
}
|
||||
|
||||
ol li,
|
||||
ul li {
|
||||
padding: 0.2rem 0;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0.75rem 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
pre {
|
||||
margin: 1rem 0;
|
||||
max-width: var(--width-card-wide);
|
||||
padding: 1rem 0;
|
||||
}
|
||||
|
||||
pre code,
|
||||
pre samp {
|
||||
display: block;
|
||||
max-width: var(--width-card-wide);
|
||||
padding: 0.5rem 2rem;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
small {
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
|
||||
sup {
|
||||
background-color: var(--color-secondary);
|
||||
border-radius: var(--border-radius);
|
||||
color: var(--color-bg);
|
||||
font-size: xx-small;
|
||||
font-weight: bold;
|
||||
margin: 0.2rem;
|
||||
padding: 0.2rem 0.3rem;
|
||||
position: relative;
|
||||
top: -2px;
|
||||
}
|
||||
/* sup::before{
|
||||
content: '[';
|
||||
}
|
||||
sup::after{
|
||||
content: ']';
|
||||
} */
|
||||
|
||||
/* Links */
|
||||
a {
|
||||
color: var(--color);
|
||||
display: inline-block;
|
||||
font-weight: bold;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
a:hover {
|
||||
color: var(--hover-color);
|
||||
/* text-decoration: underline; */
|
||||
}
|
||||
a:hover sup{
|
||||
background-color: var(--hover-color);
|
||||
}
|
||||
|
||||
a b,
|
||||
a em,
|
||||
a i,
|
||||
a strong,
|
||||
button {
|
||||
border-radius: var(--border-radius);
|
||||
display: inline-block;
|
||||
font-size: medium;
|
||||
font-weight: bold;
|
||||
line-height: var(--line-height);
|
||||
margin: 0.5rem 0;
|
||||
padding: 1rem 2rem;
|
||||
}
|
||||
|
||||
button {
|
||||
font-family: var(--font-family);
|
||||
}
|
||||
|
||||
button:hover {
|
||||
cursor: pointer;
|
||||
color: var(--hover-color);
|
||||
}
|
||||
|
||||
a b,
|
||||
a strong,
|
||||
button {
|
||||
background-color: var(--color);
|
||||
border: 2px solid var(--color);
|
||||
color: var(--color-bg);
|
||||
}
|
||||
|
||||
a em,
|
||||
a i {
|
||||
border: 2px solid var(--color);
|
||||
border-radius: var(--border-radius);
|
||||
color: var(--color);
|
||||
display: inline-block;
|
||||
padding: 1rem 2rem;
|
||||
}
|
||||
|
||||
article aside a {
|
||||
color: var(--color-secondary);
|
||||
}
|
||||
|
||||
/* Images */
|
||||
figure {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
figure img {
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
figure figcaption {
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
|
||||
/* Forms */
|
||||
|
||||
button:disabled,
|
||||
input:disabled {
|
||||
background: var(--color-bg-secondary);
|
||||
border-color: var(--color-bg-secondary);
|
||||
color: var(--color-text-secondary);
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
button[disabled]:hover {
|
||||
filter: none;
|
||||
}
|
||||
|
||||
form {
|
||||
border: 1px solid var(--color-bg-secondary);
|
||||
border-radius: var(--border-radius);
|
||||
box-shadow: var(--box-shadow) var(--color-shadow);
|
||||
display: block;
|
||||
max-width: var(--width-card-wide);
|
||||
min-width: var(--width-card);
|
||||
padding: 1.5rem;
|
||||
text-align: var(--justify-normal);
|
||||
}
|
||||
|
||||
form header {
|
||||
margin: 1.5rem 0;
|
||||
padding: 1.5rem 0;
|
||||
}
|
||||
|
||||
input,
|
||||
label,
|
||||
select,
|
||||
textarea {
|
||||
display: block;
|
||||
font-size: inherit;
|
||||
max-width: var(--width-card-wide);
|
||||
}
|
||||
|
||||
input[type="checkbox"],
|
||||
input[type="radio"] {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
input[type="checkbox"]+label,
|
||||
input[type="radio"]+label {
|
||||
display: inline-block;
|
||||
font-weight: normal;
|
||||
position: relative;
|
||||
top: 1px;
|
||||
}
|
||||
|
||||
input,
|
||||
select,
|
||||
textarea {
|
||||
border: 1px solid var(--color-bg-secondary);
|
||||
border-radius: var(--border-radius);
|
||||
margin-bottom: 1rem;
|
||||
padding: 0.4rem 0.8rem;
|
||||
}
|
||||
|
||||
input[readonly],
|
||||
textarea[readonly] {
|
||||
background-color: var(--color-bg-secondary);
|
||||
}
|
||||
|
||||
label {
|
||||
font-weight: bold;
|
||||
margin-bottom: 0.2rem;
|
||||
}
|
||||
|
||||
/* Tables */
|
||||
table {
|
||||
border: 1px solid var(--color-bg-secondary);
|
||||
border-radius: var(--border-radius);
|
||||
border-spacing: 0;
|
||||
display: inline-block;
|
||||
max-width: 100%;
|
||||
overflow-x: auto;
|
||||
padding: 0;
|
||||
/* white-space: nowrap; */
|
||||
}
|
||||
|
||||
table td,
|
||||
table th,
|
||||
table tr {
|
||||
padding: 0.4rem 0.8rem;
|
||||
text-align: var(--justify-important);
|
||||
}
|
||||
|
||||
table thead {
|
||||
background-color: var(--color);
|
||||
border-collapse: collapse;
|
||||
border-radius: var(--border-radius);
|
||||
color: var(--color-bg);
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
table thead th:first-child {
|
||||
border-top-left-radius: var(--border-radius);
|
||||
}
|
||||
|
||||
table thead th:last-child {
|
||||
border-top-right-radius: var(--border-radius);
|
||||
}
|
||||
|
||||
table thead th:first-child,
|
||||
table tr td:first-child {
|
||||
text-align: var(--justify-normal);
|
||||
}
|
||||
|
||||
table tr:nth-child(even) {
|
||||
background-color: var(--color-accent);
|
||||
}
|
||||
|
||||
/* Quotes */
|
||||
blockquote {
|
||||
display: block;
|
||||
font-size: 100%;
|
||||
line-height: var(--line-height);
|
||||
margin: 1rem auto;
|
||||
max-width: var(--width-card-medium);
|
||||
padding: 1.5rem 1rem;
|
||||
text-align: var(--justify-important);
|
||||
}
|
||||
|
||||
blockquote footer {
|
||||
color: var(--color-text-secondary);
|
||||
display: block;
|
||||
font-size: small;
|
||||
line-height: var(--line-height);
|
||||
padding: 1.5rem 0;
|
||||
}
|
||||
|
||||
|
||||
h1.title{
|
||||
font-size: 50px;
|
||||
padding: 20vh 0;
|
||||
}
|
||||
h1.Title{
|
||||
font-size: xx-large;
|
||||
padding: 20vh 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.keypoints{
|
||||
background-color: black;
|
||||
color: var(--color-bg-secondary);
|
||||
padding: 1rem;
|
||||
|
||||
}
|
||||
.keypoints > p > strong{
|
||||
margin-left:2.5rem;
|
||||
text-transform: uppercase;
|
||||
}
|
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue