importer for atlas of surveillance data export
This commit is contained in:
Ruben van de Ven
parent
1ec45ed016
commit
a4f98a6492
1 changed files with 453 additions and 0 deletions
453
csv_importer_atlas-of-surveillance.py
Normal file
453
csv_importer_atlas-of-surveillance.py
Normal file
|
|
@ -0,0 +1,453 @@
|
|||
from time import sleep
|
||||
from typing import Optional
|
||||
import urllib.request
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
import argparse
|
||||
import datetime
|
||||
import tqdm
|
||||
import csv
|
||||
from geopy.geocoders import Nominatim
|
||||
|
||||
logger = logging.getLogger('wiki.importer')
|
||||
|
||||
default_categories = [
|
||||
'Person',
|
||||
'Institution',
|
||||
'Technology',
|
||||
'Deployments',
|
||||
'Dataset',
|
||||
'City',
|
||||
'Country',
|
||||
]
|
||||
|
||||
|
||||
geocoder = Nominatim(user_agent="tutorial")
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Turn wiki into nodes & links, usable by d3-force.')
|
||||
parser.add_argument('--categories', metavar='categories', default=default_categories, nargs='+',
|
||||
help='Categories')
|
||||
parser.add_argument('--url', default="https://www.securityvision.io/wiki/api.php",
|
||||
help='Wiki API URL')
|
||||
parser.add_argument('--output', default="semantic_data.json",
|
||||
help='Output JSON file')
|
||||
parser.add_argument('--credentials', default="no_credentials.json",
|
||||
help="JSON file containing the Bot's credentials")
|
||||
parser.add_argument('--csv', default="Atlas of Surveillance-Gunshot Detection,Face Recognition,Real-Time Crime Center,Video Analytics-20220621.csv",
|
||||
help="CVS file to import")
|
||||
parser.add_argument('--citeref', default="atlasofsurveillance2022",
|
||||
help="Bibliography key for imported items")
|
||||
parser.add_argument('--dry-run', '-n', action="store_true",
|
||||
help="Dry run")
|
||||
parser.add_argument('--skip-geolocation', action="store_true",
|
||||
help="Skip geolocation fetch, for faster dry-run")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.skip_geolocation and not args.dry_run:
|
||||
raise Exception("Cannot do a real run without geolocating cities")
|
||||
|
||||
with open(args.credentials) as fp:
|
||||
credentials = json.load(fp)
|
||||
username = credentials['user']
|
||||
password = credentials['password']
|
||||
|
||||
|
||||
def get_session():
|
||||
S = requests.Session()
|
||||
|
||||
URL = args.url
|
||||
|
||||
# Retrieve login token first
|
||||
PARAMS_0 = {
|
||||
'action': "query",
|
||||
'meta': "tokens",
|
||||
'type': "login",
|
||||
'format': "json"
|
||||
}
|
||||
|
||||
R = S.get(url=URL, params=PARAMS_0)
|
||||
DATA = R.json()
|
||||
logger.debug(DATA)
|
||||
LOGIN_TOKEN = DATA['query']['tokens']['logintoken']
|
||||
|
||||
logger.debug(LOGIN_TOKEN)
|
||||
|
||||
# Send a post request to login. Using the main account for login is not
|
||||
# supported. Obtain credentials via Special:BotPasswords
|
||||
# (https://www.mediawiki.org/wiki/Special:BotPasswords) for lgname & lgpassword
|
||||
|
||||
PARAMS_1 = {
|
||||
'action': "login",
|
||||
'lgname': username,
|
||||
'lgpassword': password,
|
||||
'lgtoken': LOGIN_TOKEN,
|
||||
'format': "json"
|
||||
}
|
||||
|
||||
R = S.post(URL, data=PARAMS_1)
|
||||
DATA = R.json()
|
||||
|
||||
logger.debug(DATA)
|
||||
if DATA['login']['result'] != 'Success':
|
||||
raise Exception("Failed logging in")
|
||||
|
||||
return S
|
||||
|
||||
|
||||
# Map columns
|
||||
[
|
||||
'AOSNUMBER',
|
||||
'City', # City_state
|
||||
# 'County',
|
||||
'State',
|
||||
'Agency', # Institution
|
||||
'Type of LEA', # Law enforment agency (Institution type?)
|
||||
'Summary', # body text
|
||||
'Type of Juris', # instution type? (municipal/county/state etc)
|
||||
'Technology', # deployment type? face recognition etc
|
||||
'Vendor', # empty or clearview ai, veritone etc. (Institution)
|
||||
'Link 1',
|
||||
# 'Link 1 Snapshot',
|
||||
'Link 1 Source',
|
||||
# 'Link 1 Type',
|
||||
'Link 1 Date',
|
||||
'Link 2',
|
||||
# 'Link 2 Snapshot',
|
||||
'Link 2 Source',
|
||||
# 'Link 2 Type',
|
||||
'Link 2 Date',
|
||||
'Link 3',
|
||||
# 'Link 3 Snapshot',
|
||||
'Link 3 Source',
|
||||
# 'Link 3 Type',
|
||||
'Link 3 Date',
|
||||
# 'Other Links',
|
||||
'Statewide Network of Agency Photos (SNAP)', # single deplyment, aggregrate Used by
|
||||
'Face Analysis Comparison & Examination System (FACES)', # single deplyment, aggregrate Used by
|
||||
'Maryland Image Repository System', # single deplyment, aggregrate Used by
|
||||
#'Clearview AI', # no aggregation
|
||||
#'BriefCam', # no aggregation?
|
||||
'FACE Services', # create link: Input for
|
||||
'Relevant for the Wiki?', # FILTER!!
|
||||
]
|
||||
|
||||
|
||||
# title: Use of [VENDOR ]TECHNOLOGY by AGENCY
|
||||
# City: CITY (STATE) or aggregated
|
||||
# Country: USA
|
||||
# Software Used: VENDOR TECHNOLOGY
|
||||
# Used by: AGENCY
|
||||
# Information Certainty: Documented
|
||||
# Input for: [FACE Services]
|
||||
# body: <blockquote> SUMMARY</blockquote>
|
||||
# Additional properties:
|
||||
# Original Sources: url date (link 1, 2, 3)
|
||||
# AOSNUMBER
|
||||
# CiteRef: args.citeref
|
||||
|
||||
# title: AGENCY
|
||||
# City: CITY (STATE)
|
||||
# Institution Type: Law Enforcement
|
||||
|
||||
|
||||
# aggregate agencies when these columns are 'yes'
|
||||
aggregates = {
|
||||
'Statewide Network of Agency Photos (SNAP)': [],
|
||||
'Face Analysis Comparison & Examination System (FACES)': [],
|
||||
'Maryland Image Repository System': [],
|
||||
}
|
||||
|
||||
institutions = {}
|
||||
cities = {}
|
||||
technologies = {}
|
||||
|
||||
def mapEntry(entry) -> Optional[dict]:
|
||||
if entry['Relevant for the Wiki?'] != 'Yes':
|
||||
logger.warning(f'Ignore entry {entry["AOSNUMBER"]}')
|
||||
return None
|
||||
else:
|
||||
hasAggregated = False
|
||||
for field in aggregates.keys():
|
||||
if entry[field] == 'Yes':
|
||||
aggregates[field].append(entry)
|
||||
hasAggregated = True
|
||||
|
||||
if hasAggregated:
|
||||
return None
|
||||
|
||||
return mapDeployment(entry)
|
||||
|
||||
def mapTechnology(entry):
|
||||
entry['Vendor'] = entry['Vendor'].strip()
|
||||
tech = {
|
||||
'title': f"{entry['Vendor'] if entry['Vendor'] else 'Unknown'} {entry['Technology']}",
|
||||
"@type": "Products",
|
||||
'properties': {
|
||||
"Developed by": entry['Vendor'],
|
||||
},
|
||||
"additionalProperties": {
|
||||
"Technology Type": entry['Technology'],
|
||||
"Needs processing of the title": "Yes",
|
||||
'CiteRef': args.citeref,
|
||||
}
|
||||
}
|
||||
technologies[tech['title']] = tech
|
||||
return tech
|
||||
|
||||
def mapDeployment(entry):
|
||||
global args
|
||||
city = mapCity(entry)
|
||||
mapInstitution(entry)
|
||||
tech = mapTechnology(entry)
|
||||
return {
|
||||
'title': f"{entry['Vendor']} {entry['Technology']} used by {entry['Agency']}".replace(' '," ").strip(),
|
||||
'@type': 'Deployments',
|
||||
'properties': {
|
||||
"Keywords": [entry['Technology']],
|
||||
"used by": entry['Agency'],
|
||||
"Software Deployed": tech['title'],
|
||||
"City": city['title'],
|
||||
"Information Certainty": "Documented",
|
||||
},
|
||||
"additionalProperties": {
|
||||
"URL": [
|
||||
"https://atlasofsurveillance.org/es/a/"+entry['AOSNUMBER'],
|
||||
entry['Link 1'],
|
||||
entry['Link 2'],
|
||||
entry['Link 3'],
|
||||
],
|
||||
'CiteRef': args.citeref,
|
||||
"Input for": "FACES (FBI) Dataset" if entry['FACE Services'] == 'Yes' else None,
|
||||
},
|
||||
"body": [entry['Summary']]
|
||||
}
|
||||
|
||||
|
||||
# Type of LEA: Court gives Institution Type::Government Instititution Sector::Justice,
|
||||
# Police/Sheriff/State Police/District Attorney/Attorney General/Prosecutor/School Police/Constables/DHS/Fusion Center/Juvenile/Security/Transit Police Type::Law Enfrocement Instititution Sector::Security
|
||||
# DMV/Emergency Services/Parks/State Agency/Transit Institution Type::Government Institution Sector::Civil Administration
|
||||
# Medical Examiner Institution Type::Government Institution Sector::Health
|
||||
# School District Institution Type::Local Government Institution Sector::Education
|
||||
# State-Local Partnership Institution Type::State-Local Partnership Institution Sector::Security
|
||||
|
||||
institution_type_sector = {
|
||||
"Court": ("Government", "Justice"),
|
||||
|
||||
"Police": ("Law Enforcement", "Security"),
|
||||
"Sheriff": ("Law Enforcement", "Security"),
|
||||
"State Police": ("Law Enforcement", "Security"),
|
||||
"District Attorney": ("Law Enforcement", "Security"),
|
||||
"Attorney General": ("Law Enforcement", "Security"),
|
||||
"Prosecutor": ("Law Enforcement", "Security"),
|
||||
"School Police": ("Law Enforcement", "Security"),
|
||||
"Constables": ("Law Enforcement", "Security"),
|
||||
"DHS": ("Law Enforcement", "Security"),
|
||||
"Fusion Center": ("Law Enforcement", "Security"),
|
||||
"Juvenile": ("Law Enforcement", "Security"),
|
||||
"Security": ("Law Enforcement", "Security"),
|
||||
"Transit Police": ("Law Enforcement", "Security"),
|
||||
"Corrections": ("Law Enforcement", "Security"),
|
||||
"Clemis": ("Law Enforcement", "Security"),
|
||||
|
||||
"DMV": ("Government", "Civil Administration"),
|
||||
"Emergency Services": ("Government", "Civil Administration"),
|
||||
"Parks": ("Government", "Civil Administration"),
|
||||
"State Agency": ("Government", "Civil Administration"),
|
||||
"Transit": ("Government", "Civil Administration"),
|
||||
|
||||
"Medical Examiner": ("Local Government", "Health"),
|
||||
"School District": ("Local Government", "Education"),
|
||||
|
||||
"State-Local Partnership": ("State-Local Partnership", "Security"),
|
||||
}
|
||||
|
||||
def mapInstitution(entry):
|
||||
# aggregate agencies as institutions from entries
|
||||
global args
|
||||
|
||||
type, sector = institution_type_sector[entry['Type of LEA']]
|
||||
info = {
|
||||
'title': entry['Agency'],
|
||||
'@type': 'Institution',
|
||||
'properties': {
|
||||
"Institution Type": type,
|
||||
"Institution Sector": sector,
|
||||
'City': mapCity(entry)['title'],
|
||||
},
|
||||
"additionalProperties": {
|
||||
"Type of Juris": entry['Type of Juris'],
|
||||
"URL": [
|
||||
"https://atlasofsurveillance.org/es/a/"+entry['AOSNUMBER'],
|
||||
entry['Link 1'],
|
||||
entry['Link 2'],
|
||||
entry['Link 3'],
|
||||
],
|
||||
'CiteRef': args.citeref,
|
||||
},
|
||||
"body": [entry['Type of LEA']],
|
||||
}
|
||||
|
||||
if entry['Agency'] in institutions:
|
||||
logger.warning(f'Ignore duplicate {entry["Agency"]}')
|
||||
else:
|
||||
institutions[entry['Agency']] = info
|
||||
|
||||
def mapCity(entry):
|
||||
title = f"{entry['City']} ({entry['State']})"
|
||||
if title not in cities:
|
||||
info = {
|
||||
'title': title,
|
||||
'@type': 'City',
|
||||
'properties': {
|
||||
"is in Country": "USA",
|
||||
},
|
||||
"additionalProperties": {
|
||||
'CiteRef': args.citeref,
|
||||
}
|
||||
}
|
||||
|
||||
if not args.skip_geolocation:
|
||||
location_response = geocoder.geocode(title + " USA")
|
||||
sleep(1) # free tier of location geocode requires 1 sec delay
|
||||
if location_response:
|
||||
location = location_response.raw
|
||||
info["properties"]["Has Coordinates"] = f"{location['lat']}, {location['lon']}"
|
||||
info["body"] = [location['display_name']]
|
||||
else:
|
||||
logger.warning(f"No location data for {title} USA")
|
||||
|
||||
cities[title] = info
|
||||
return cities[title]
|
||||
|
||||
|
||||
def mapAggregate(title, data):
|
||||
urls = [ "https://atlasofsurveillance.org/es/a/"+entry['AOSNUMBER'] for entry in data]
|
||||
urls.extend([entry['Link 1'] for entry in data])
|
||||
urls.extend([entry['Link 2'] for entry in data])
|
||||
urls.extend([entry['Link 3'] for entry in data])
|
||||
urls = list(dict.fromkeys(urls)) # unique
|
||||
urls = list(filter(lambda url: url and len(url) > 0, urls))
|
||||
|
||||
for entry in data:
|
||||
mapInstitution(entry)
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
'@type': 'Deployments',
|
||||
'properties': {
|
||||
"Information Certainty": "Documented",
|
||||
"used by": [entry['Agency'] for entry in data]
|
||||
},
|
||||
"additionalProperties": {
|
||||
"URL": urls,
|
||||
'CiteRef': args.citeref,
|
||||
}
|
||||
}
|
||||
|
||||
def renderPage(data):
|
||||
global args
|
||||
|
||||
page = f"{{{{{data['@type']}"
|
||||
for key, value in data['properties'].items():
|
||||
page += f"\n|{key}=" + (', '.join(value) if isinstance(value, list) else value)
|
||||
page += "}}\n\n"
|
||||
|
||||
if 'body' in data:
|
||||
for b in data['body']:
|
||||
if b and len(b):
|
||||
page += f"<blockquote>{b} [[CiteRef::{args.citeref}]]</blockquote>\n\n"
|
||||
|
||||
if len(data['additionalProperties']):
|
||||
page += "=== Additional properties ===\n\n"
|
||||
for key, value in data['additionalProperties'].items():
|
||||
if not isinstance(value, list):
|
||||
value = [value]
|
||||
|
||||
for v in value:
|
||||
if v:
|
||||
page += f"* {key} [[{key}::{v}]]\n"
|
||||
return page
|
||||
|
||||
def saveIfNotExists(data, page, session, token):
|
||||
# https://en.wikipedia.org/w/api.php?action=query&prop=info&titles=New%20York%20Yankeesdfsdf
|
||||
# baseurl = f"{args.url}?action=query&list=categorymembers&cmtitle=Category:{category}&format=json"
|
||||
params = {
|
||||
'action': 'edit',
|
||||
'createonly': '1',
|
||||
'title': data['title'].strip(),
|
||||
'contentformat': 'text/x-wiki',
|
||||
'text': page,
|
||||
'format': 'json',
|
||||
'token': token,
|
||||
}
|
||||
logger.debug(args.url, params)
|
||||
|
||||
if not args.dry_run:
|
||||
logger.warning(f"Creating '{data['title'].strip()}' type {data['@type']}")
|
||||
response = session.post(args.url, data=params)
|
||||
resp = response.json()
|
||||
|
||||
if 'warnings' in resp:
|
||||
logger.warning(resp)
|
||||
|
||||
logger.debug(resp)
|
||||
else:
|
||||
logger.warning(f"'{data['title'].strip()}' type {data['@type']}")
|
||||
|
||||
|
||||
def getEditToken(session):
|
||||
params = {
|
||||
'action': "query",
|
||||
'meta': "tokens",
|
||||
'type': "csrf",
|
||||
'format': "json"
|
||||
}
|
||||
|
||||
R = session.get(args.url, params=params)
|
||||
DATA = R.json()
|
||||
logger.debug(DATA)
|
||||
return DATA['query']['tokens']['csrftoken']
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger.setLevel(logging.DEBUG)
|
||||
session = get_session()
|
||||
token = getEditToken(session)
|
||||
|
||||
parsedData=[]
|
||||
with open(args.csv, newline='') as csvfile:
|
||||
csvreader = csv.DictReader(csvfile, delimiter=',')
|
||||
for row in csvreader:
|
||||
data = mapEntry(row)
|
||||
if data is None:
|
||||
continue
|
||||
parsedData.append(data)
|
||||
|
||||
parsedData.extend([mapAggregate(title, a) for title, a in aggregates.items()])
|
||||
parsedData.extend(cities.values())
|
||||
parsedData.extend(technologies.values())
|
||||
parsedData.extend(institutions.values())
|
||||
# print(parsedData)
|
||||
|
||||
for i, data in enumerate(parsedData):
|
||||
page = renderPage(data)
|
||||
# if data['@type'] == "Institution":
|
||||
# print(data['title'])
|
||||
# print(page)
|
||||
# break
|
||||
saveIfNotExists(data, page, session, token)
|
||||
|
||||
# if i > 5:
|
||||
# break
|
||||
|
||||
print(f"total: {len(parsedData)} items (of which {len(institutions)} institutions, {len(cities)} cities, {len(technologies)} products)")
|
||||
print (len(parsedData) - len(institutions) - len(cities) - len(technologies), "deployments" )
|
||||
|
||||
|
||||
|
||||
#
|
||||
# Title: vendor/unknown
|
||||
|
||||
# Postprocessing: make sure unknown are numbered, and multiple related deployments individual unknowns are created.
|
||||
Loading…
Reference in a new issue