689 lines
39 KiB
HTML
689 lines
39 KiB
HTML
|
<!DOCTYPE html>
|
|||
|
|
<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang="">
|
|||
|
|
|
|||
|
|
<head>
|
|||
|
|
<meta charset="utf-8" />
|
|||
|
|
<meta name="generator" content="pandoc" />
|
|||
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
|
|||
|
|
<meta name="author" content="Ruben van de Ven, Ildikó Zonga Plájás, Cyan Bae, Francesco Ragazzi" />
|
|||
|
|
<title>Algorithmic Security Vision: Diagrams of Computer Vision Politics</title>
|
|||
|
|
<style>
|
|||
|
|
/* div[data-custom-style='Body Text']{
|
|||
|
|
background: rgba(255,255,255,.5)
|
|||
|
|
} */
|
|||
|
|
code {
|
|||
|
|
white-space: pre-wrap;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
span.smallcaps {
|
|||
|
|
font-variant: small-caps;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
span.underline {
|
|||
|
|
text-decoration: underline;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
div.column {
|
|||
|
|
display: inline-block;
|
|||
|
|
vertical-align: top;
|
|||
|
|
width: 50%;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
div.hanging-indent {
|
|||
|
|
margin-left: 1.5em;
|
|||
|
|
text-indent: -1.5em;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
ul.task-list {
|
|||
|
|
list-style: none;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
.display.math {
|
|||
|
|
display: block;
|
|||
|
|
text-align: center;
|
|||
|
|
margin: 0.5rem auto;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/*Filenames with code blocks: https://stackoverflow.com/a/58199362*/
|
|||
|
|
div.sourceCode::before {
|
|||
|
|
content: attr(data-filename);
|
|||
|
|
display: block;
|
|||
|
|
background-color: #cfeadd;
|
|||
|
|
font-family: monospace;
|
|||
|
|
font-weight: bold;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#collage {
|
|||
|
|
position: fixed;
|
|||
|
|
z-index: -1;
|
|||
|
|
background-color: white;
|
|||
|
|
left: 0;
|
|||
|
|
top: 0;
|
|||
|
|
right: 0;
|
|||
|
|
bottom: 0;
|
|||
|
|
overflow: hidden;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
#collage_window {
|
|||
|
|
position: absolute;
|
|||
|
|
top: -1000px;
|
|||
|
|
left: -1000px;
|
|||
|
|
}
|
|||
|
|
</style>
|
|||
|
|
<link rel="stylesheet" href="paper.css" />
|
|||
|
|
<script src="assets/wNumb-1.2.0.min.js"></script>
|
|||
|
|
<script src="assets/annotate.js"></script>
|
|||
|
|
<script>
|
|||
|
|
const centerPoints = [
|
|||
|
|
[2759, 6452],
|
|||
|
|
[14335, 5364],
|
|||
|
|
[5757, 10084],
|
|||
|
|
[7137, 3869], // left in practice is -5746px;, top: -2988px;:
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
const canvasCenter = [20077 / 2, 10331 / 2]
|
|||
|
|
|
|||
|
|
let scale = .5;
|
|||
|
|
|
|||
|
|
|
|||
|
|
function easeInOutQuart(x) {
|
|||
|
|
return x < 0.5 ? 8 * x * x * x * x : 1 - Math.pow(-2 * x + 2, 4) / 2;
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
let scroll_offsets = []
|
|||
|
|
|
|||
|
|
function calculateScrollOffsets() {
|
|||
|
|
const anchorEls = document.getElementsByClassName('anchor');
|
|||
|
|
|
|||
|
|
offsets = []
|
|||
|
|
|
|||
|
|
for (let anchorEl of anchorEls) {
|
|||
|
|
const align_pos = centerPoints[anchorEl.dataset.i];
|
|||
|
|
const bbox = anchorEl.getBoundingClientRect()
|
|||
|
|
const scroll_y = bbox.top + (bbox.height / 2) + window.scrollY;
|
|||
|
|
offsets.push([scroll_y, anchorEl.dataset.i])
|
|||
|
|
}
|
|||
|
|
return offsets.sort((a, b) => a[0] - b[0]);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
window.addEventListener('DOMContentLoaded', () => {
|
|||
|
|
|
|||
|
|
scroll_offsets = calculateScrollOffsets()
|
|||
|
|
|
|||
|
|
const windowEl = document.getElementById('collage_window')
|
|||
|
|
|
|||
|
|
|
|||
|
|
function updateScroll() {
|
|||
|
|
center_y = window.scrollY + window.innerHeight / 2
|
|||
|
|
prev = null;
|
|||
|
|
next = null;
|
|||
|
|
for (let offset of scroll_offsets) {
|
|||
|
|
if (offset[0] > center_y) {
|
|||
|
|
next = offset
|
|||
|
|
break;
|
|||
|
|
}
|
|||
|
|
prev = offset
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
let source_pos, target_pos, source_scale, target_scale;
|
|||
|
|
|
|||
|
|
if (prev === null) {
|
|||
|
|
prev = [next[0] - window.innerHeight / 2, null]
|
|||
|
|
|
|||
|
|
source_scale = .1
|
|||
|
|
target_scale = .45
|
|||
|
|
source_pos = canvasCenter
|
|||
|
|
target_pos = centerPoints[next[1]]
|
|||
|
|
|
|||
|
|
} else if (next === null) {
|
|||
|
|
next = [prev[0] + window.innerHeight / 2, null]
|
|||
|
|
|
|||
|
|
source_scale = .45
|
|||
|
|
target_scale = .1
|
|||
|
|
source_pos = centerPoints[prev[1]]
|
|||
|
|
target_pos = canvasCenter
|
|||
|
|
} else {
|
|||
|
|
source_pos = centerPoints[prev[1]]
|
|||
|
|
target_pos = centerPoints[next[1]]
|
|||
|
|
target_scale = .45
|
|||
|
|
source_scale = .45
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
const t = Math.min(1, Math.max(0, (center_y - prev[0]) / (next[0] - prev[0])))
|
|||
|
|
t_ease = easeInOutQuart(t)
|
|||
|
|
|
|||
|
|
const dx = target_pos[0] - source_pos[0];
|
|||
|
|
const dy = target_pos[1] - source_pos[1];
|
|||
|
|
const ds = target_scale - source_scale
|
|||
|
|
|
|||
|
|
// console.log('twean from', source_pos, 'to', target_pos, 't', t_ease)
|
|||
|
|
// console.log('twean scale', source_scale, 'to', target_scale, 't', t_ease)
|
|||
|
|
|
|||
|
|
|
|||
|
|
scale = source_scale + t_ease * ds;
|
|||
|
|
x = -1 * (source_pos[0] + dx * t_ease) * scale + window.innerWidth / 2;
|
|||
|
|
y = -1 * (source_pos[1] + dy * t_ease) * scale + window.innerHeight / 2;
|
|||
|
|
// console.log('x', x, 'y', y, 'scale', scale)
|
|||
|
|
|
|||
|
|
// console.log(x, y);
|
|||
|
|
|
|||
|
|
windowEl.style.transform = `scale(${scale})`
|
|||
|
|
windowEl.style.left = `${x}px`
|
|||
|
|
windowEl.style.top = `${y}px`
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
windowEl.style.transform = `scale(${scale})`
|
|||
|
|
|
|||
|
|
window.addEventListener('resize', (ev) => {
|
|||
|
|
scroll_offsets = calculateScrollOffsets()
|
|||
|
|
updateScroll()
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
window.addEventListener('scroll', updateScroll)
|
|||
|
|
|
|||
|
|
updateScroll()
|
|||
|
|
|
|||
|
|
let options = {
|
|||
|
|
// root: document.querySelector("#scrollArea"), // viewport by default
|
|||
|
|
rootMargin: `${-Math.round(innerHeight/3)}px`, //"0px",
|
|||
|
|
threshold: 0,
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
let observer = new IntersectionObserver((entries, observer) => {
|
|||
|
|
entries.forEach((entry) => {
|
|||
|
|
index = entry.target.dataset.i;
|
|||
|
|
console.log(entry)
|
|||
|
|
if(index >= windowEl.children.length) {
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
playerEl = windowEl.children[index];
|
|||
|
|
if(entry.isIntersecting){
|
|||
|
|
playerEl.classList.add('active')
|
|||
|
|
} else {
|
|||
|
|
playerEl.classList.remove('active')
|
|||
|
|
}
|
|||
|
|
})
|
|||
|
|
}, options);
|
|||
|
|
|
|||
|
|
const anchorEls = document.getElementsByClassName('anchor');
|
|||
|
|
for (const anchorEl of anchorEls) {
|
|||
|
|
observer.observe(anchorEl)
|
|||
|
|
}
|
|||
|
|
console.log(anchorEls)
|
|||
|
|
// .forEach(el => observer.observe());
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
// console.log(anchorEl.dataset.title);
|
|||
|
|
// const toSelect = typeof anchorEl.dataset.title == 'undefined' || anchorEl.dataset.title == 'none' ? null : frameEl.contentWindow.getIdForTitle(anchorEl.dataset.title);
|
|||
|
|
// // navItemEl.hash url-encodes
|
|||
|
|
// // let targetEl = document.getElementById(navItemEl.attributes.href.value.substr(1));
|
|||
|
|
// // let wrapperEl = targetEl.parentNode;
|
|||
|
|
// let intersectionObserver = new IntersectionObserver(function (entries) {
|
|||
|
|
// console.log(entries);
|
|||
|
|
// // If intersectionRatio is 0, the target is out of view
|
|||
|
|
// // and we do not need to do anything.
|
|||
|
|
// // if (entries[0].intersectionRatio <= 0) {
|
|||
|
|
// // // navItemEl.classList.remove('active');
|
|||
|
|
// // } else {
|
|||
|
|
// // if (toSelect === null) {
|
|||
|
|
// // frameEl.contentWindow.mapGraph.triggerReset();
|
|||
|
|
// // // frameEl.contentWindow.mapGraph.deselectNode();
|
|||
|
|
// // // frameEl.contentWindow.mapGraph.resetZoom();
|
|||
|
|
// // } else {
|
|||
|
|
// // frameEl.contentWindow.mapGraph.triggerSelect(toSelect);
|
|||
|
|
// // // frameEl.contentWindow.mapGraph.selectNode(node);
|
|||
|
|
// // }
|
|||
|
|
// // // navItemEl.classList.add('active');
|
|||
|
|
// // }
|
|||
|
|
|
|||
|
|
// });
|
|||
|
|
// // start observing
|
|||
|
|
// intersectionObserver.observe(anchorEl);
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
|
|||
|
|
// const linkEls = document.getElementsByClassName('maplink');
|
|||
|
|
// for (let linkEl of linkEls) {
|
|||
|
|
// linkEl.addEventListener('click', (ev) => {
|
|||
|
|
// const toSelect = typeof linkEl.dataset.title == 'undefined' || linkEl.dataset.title == 'none' ? null : frameEl.contentWindow.getIdForTitle(linkEl.dataset.title);
|
|||
|
|
|
|||
|
|
// if (toSelect === null) {
|
|||
|
|
// frameEl.contentWindow.mapGraph.deselectNode();
|
|||
|
|
// frameEl.contentWindow.mapGraph.resetZoom();
|
|||
|
|
// } else {
|
|||
|
|
// const node = frameEl.contentWindow.mapGraph.graph.nodes.filter(n => n.id == toSelect)[0]
|
|||
|
|
// frameEl.contentWindow.mapGraph.selectNode(node);
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
// })
|
|||
|
|
// linkEl.addEventListener('mouseover', (ev) => {
|
|||
|
|
// const toSelect = typeof linkEl.dataset.title == 'undefined' || linkEl.dataset.title == 'none' ? null : frameEl.contentWindow.getIdForTitle(linkEl.dataset.title);
|
|||
|
|
// if (toSelect) {
|
|||
|
|
|
|||
|
|
// const node = frameEl.contentWindow.mapGraph.graph.nodes.filter(n => n.id == toSelect)[0]
|
|||
|
|
// frameEl.contentWindow.mapGraph.hoverNode(false, node);
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
// })
|
|||
|
|
// linkEl.addEventListener('mouseout', (ev) => {
|
|||
|
|
// const toSelect = typeof linkEl.dataset.title == 'undefined' || linkEl.dataset.title == 'none' ? null : frameEl.contentWindow.getIdForTitle(linkEl.dataset.title);
|
|||
|
|
// if (toSelect) {
|
|||
|
|
// const node = frameEl.contentWindow.mapGraph.graph.nodes.filter(n => n.id == toSelect)[0]
|
|||
|
|
// frameEl.contentWindow.mapGraph.endHoverNode(node);
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
// })
|
|||
|
|
|
|||
|
|
// }
|
|||
|
|
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
</script>
|
|||
|
|
</head>
|
|||
|
|
|
|||
|
|
<body>
|
|||
|
|
<header id="title-block-header">
|
|||
|
|
<h1 class="title">Algorithmic Security Vision: Diagrams of Computer
|
|||
|
|
Vision Politics</h1>
|
|||
|
|
<p class="author"><em>Ruben van de Ven, Ildikó Zonga Plájás, Cyan Bae,
|
|||
|
|
Francesco Ragazzi</em></p>
|
|||
|
|
<p class="date">December 2023</p>
|
|||
|
|
</header>
|
|||
|
|
<div id='collage'>
|
|||
|
|
<div id="collage_window">
|
|||
|
|
<!--data-poster-url="annotation-BIXG4VTL.svg"-->
|
|||
|
|
<annotation-player style="position:absolute;width:3243px; height:2635px;left:514px;top:6329px;"
|
|||
|
|
data-url-prefix="assets" stroke="blue" data-crop='whole'
|
|||
|
|
data-annotation-url="annotation-F19O9PGE.json"></annotation-player>
|
|||
|
|
<annotation-player style="position:absolute;width:11867px;height:2753px;left:3905px;top:4333px;"
|
|||
|
|
data-url-prefix="assets" stroke="blue" data-crop='whole'
|
|||
|
|
data-annotation-url="annotation-XYGE65SB.json"></annotation-player>
|
|||
|
|
<annotation-player style="position:absolute;width:4450px;height:3250px;left:5056px;top:7081px;"
|
|||
|
|
data-url-prefix="assets" stroke="blue" data-crop='whole'
|
|||
|
|
data-annotation-url="annotation-NKOUGNWE.json"></annotation-player>
|
|||
|
|
<annotation-player style="position:absolute;width:2547px; height:2710px; left:5690px;top:1584px;"
|
|||
|
|
data-url-prefix="assets" stroke="blue" data-crop='whole'
|
|||
|
|
data-annotation-url="annotation-J4AOBZCJ.json"></annotation-player>
|
|||
|
|
<annotation-player style="position:absolute;width:2952px; height:5043px;left:0;top:0;"
|
|||
|
|
data-url-prefix="assets" stroke="blue" data-crop='whole'
|
|||
|
|
data-annotation-url="annotation-BIXG4VTL.json"></annotation-player>
|
|||
|
|
<!--data-poster-url="annotation-F19O9PGE.svg"-->
|
|||
|
|
<!--data-poster-url="annotation-XYGE65SB.svg"-->
|
|||
|
|
<!--data-poster-url="annotation-NKOUGNWE.svg"-->
|
|||
|
|
<!--data-poster-url="annotation-J4AOBZCJ.svg"-->
|
|||
|
|
</div>
|
|||
|
|
</div>
|
|||
|
|
<section id="part1">
|
|||
|
|
<section id="managing-error-from-the-sublime-to-the-risky-algorithm" class="level2">
|
|||
|
|
<h2>3. Managing error: from the sublime to the risky algorithm</h2>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>Our third emerging figuration concerns the place of the error. A
|
|||
|
|
large body of literature examines actual and speculative cases of
|
|||
|
|
algorithmic prediction based on self-learning systems (Azar et al.,
|
|||
|
|
2021). Central to these analyses is the boundary-drawing performed by
|
|||
|
|
such algorithmic devices, enacting (in)security by rendering their
|
|||
|
|
subjects as more- or less-risky others (Amicelle et al., 2015: 300;
|
|||
|
|
Amoore and De Goede, 2005; Aradau et al., 2008; Aradau and Blanke, 2018)
|
|||
|
|
based on a spectrum of individual and environmental features (Calhoun,
|
|||
|
|
2023). In other words, these predictive devices conceptualize risk as
|
|||
|
|
something produced by, and thus external to, security technologies.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>In this critical literature on algorithmic practices, practitioners
|
|||
|
|
working with algorithmic technologies are often critiqued for
|
|||
|
|
understanding software as “sublime” (e.g. Wilcox, 2017: 3). However, in
|
|||
|
|
our diagrams, algorithmic vision appears as a practice of managing
|
|||
|
|
error. The practitioners we interviewed are aware of the error-prone
|
|||
|
|
nature of their systems but know it will never be perfect, and see it as
|
|||
|
|
a key metric that needs to be acted upon.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>The most prominent way in which error figures in the diagrams is in
|
|||
|
|
its quantified form of the true positive and false positive rates, TPR
|
|||
|
|
and FPR. The significance and definition of these metrics is stressed by
|
|||
|
|
CTO Gerwin van der Lugt (Diagram 6). In camera surveillance, the false
|
|||
|
|
positive rate could be described as the number of fales positive
|
|||
|
|
classifications relative to the number of video frames being analyzed.
|
|||
|
|
Upon writing down these definitions, van der Lugt corrected his initial
|
|||
|
|
definitions, as these definitions determine the work of his development
|
|||
|
|
team, the ways in which his clients — security operators — engage with
|
|||
|
|
the technology, and whether they perceive the output of the system as
|
|||
|
|
trustworthy.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Figure">
|
|||
|
|
<div class="anchor" data-i="0" style="height:2.3in"></div>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Caption">
|
|||
|
|
<p>Diagram 6. Gerwin van der Lugt corrects his initial definitions of
|
|||
|
|
the true positive and false positive rates, and stresses the importance
|
|||
|
|
of their precise definition.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>The figuration of algorithmic security vision as inherently imprecise
|
|||
|
|
affects the operationalization of security practices. Van der Lugt’s
|
|||
|
|
example concerns whether the violence detection algorithm developed by
|
|||
|
|
Oddity.ai should be trained to categorize friendly fighting
|
|||
|
|
(<em>stoeien</em>) between friends as “violence” or not. In this
|
|||
|
|
context, van der Lugt finds it important to differentiate what counts as
|
|||
|
|
false positive in the algorithm’s evaluation metric from an error in the
|
|||
|
|
algorithm’s operationalization of a security question.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>He gives two reasons to do so. First, he anticipates that the
|
|||
|
|
exclusion of <em>stoeien</em> from the category of violence would
|
|||
|
|
negatively impact TPR. In the iterative development of self-learning
|
|||
|
|
systems, the TPR and FPR, together with the true and false
|
|||
|
|
<em>negative</em> rates must perform a balancing act. Van der Lugt
|
|||
|
|
outlines that with their technology they aim for fewer than 100 false
|
|||
|
|
positives per 100 million frames per week. The FPR becomes indicative of
|
|||
|
|
the algorithm’s quality, as too many faulty predictions will desensitize
|
|||
|
|
the human operator to system alerts.
|
|||
|
|
</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>This leads to van der Lugt’s second point: He fears that the
|
|||
|
|
exclusion of <em>stoeien</em> from the violence category might cause
|
|||
|
|
unexpected biases in the system. For example, instead of distinguishing
|
|||
|
|
violence from <em>stoeien</em> based on people’s body movements, the
|
|||
|
|
algorithm might make the distinction based on their age. For van der
|
|||
|
|
Lugt, this would be an undesirable and hard to notice form of
|
|||
|
|
discrimination. In developing algorithmic (in)security, error is figured
|
|||
|
|
not merely as a mathematical concept but (as shown in Diagram 6) as a
|
|||
|
|
notion that invites pre-emption — a mitigation of probable failure — for
|
|||
|
|
which the developer is responsible. The algorithmic condition of
|
|||
|
|
security vision is figured as the pre-emption of error.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Figure">
|
|||
|
|
<div class="anchor" data-i="1" style="height:6in"></div>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Caption">
|
|||
|
|
<p>Diagram 7. By drawing errors on a timeline, van Rest calls attention
|
|||
|
|
to the pre-emptive nature of error in the development process of
|
|||
|
|
computer vision technologies.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>According to critical AI scholar Matteo Pasquinelli, “machine
|
|||
|
|
learning is technically based on formulas for error correction” (2019:
|
|||
|
|
2). Therefore, any critical engagement with such algorithmic processes
|
|||
|
|
needs to go beyond citing errors, “for it is precisely through these
|
|||
|
|
variations that the algorithm learns what to do” (Amoore, 2019: 164),
|
|||
|
|
pushing us to reconsider any argument based on the inaccuracy of the
|
|||
|
|
systems.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>The example of <em>stoeien</em> suggests that it is not so much a
|
|||
|
|
question if, or how much, these algorithms err, but how these errors are
|
|||
|
|
anticipated and negotiated. Thus, taking error as a hallmark of machine
|
|||
|
|
learning we can see how practices of (in)security become shaped by the
|
|||
|
|
notion of mathematical error well beyond their development stages. Error
|
|||
|
|
figures centrally in the development, acquisition and deployment of such
|
|||
|
|
devices. As one respondent indicated, predictive devices are inherently
|
|||
|
|
erroneous, but the quantification of their error makes them amenable to
|
|||
|
|
"risk management.”</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>While much has been written about security technologies as a device
|
|||
|
|
<em>for</em> risk management, little is known about how security
|
|||
|
|
technologies are conceptualized as objects <em>of</em> risk management.
|
|||
|
|
What happens then in this double relation of risk? The figure of the
|
|||
|
|
error enters the diagrams as a mathematical concept, throughout the
|
|||
|
|
conversations we see its figure permeate the discourse around
|
|||
|
|
algorithmic security vision. By figuring algorithmic security vision
|
|||
|
|
through the notion of error, risk is placed at the heart of the security
|
|||
|
|
apparatus.
|
|||
|
|
</p>
|
|||
|
|
</div>
|
|||
|
|
</section>
|
|||
|
|
</section>
|
|||
|
|
<section id="con-figurations-of-algorithmic-security-vision-fragmenting-accountability-and-expertise"
|
|||
|
|
class="level1">
|
|||
|
|
<h1>Con-figurations of algorithmic security vision: fragmenting
|
|||
|
|
accountability and expertise</h1>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>In the previous section we explored the changing <em>figurations</em>
|
|||
|
|
of key dimensions of algorithmic security vision, in this section we
|
|||
|
|
examine how these figurations <em>configure</em>. For Suchman, working
|
|||
|
|
with configurations highlights “the histories and encounters through
|
|||
|
|
which things are figured <em>into meaningful existence</em>, fixing them
|
|||
|
|
through reiteration but also always engaged in ‘the perpetuity of coming
|
|||
|
|
to be’ that characterizes the biographies of objects as well as
|
|||
|
|
subjects” (Suchman, 2012: 50, emphasis ours) In other words, we are
|
|||
|
|
interested in the practices and tensions that emerge as figurations
|
|||
|
|
become embedded in material practices. We focus on two con-figurations
|
|||
|
|
that emerged in the interviews: the delegation of accountability to
|
|||
|
|
externally managed benchmarks, and the displacement of responsibility
|
|||
|
|
through the reconfiguration of the human-in-the-loop.</p>
|
|||
|
|
</div>
|
|||
|
|
<section id="delegating-accountability-to-benchmarks" class="level2">
|
|||
|
|
<h2>Delegating accountability to benchmarks</h2>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>The first configuration is related to the evaluation of the error
|
|||
|
|
rate in the training of algorithmic vision systems: it involves
|
|||
|
|
datasets, benchmark institutions, and the idea of fairness as equal
|
|||
|
|
representation among different social groups. Literature on the ethical
|
|||
|
|
and political effects of algorithmic vision has notoriously focused on
|
|||
|
|
the distribution of errors, raising questions of ethnic and racial bias
|
|||
|
|
(e.g. Buolamwini and Gebru, 2018). Our interviews reflect the concerns
|
|||
|
|
of much of this literature as the pre-emption of error figured
|
|||
|
|
repeatedly in relation to the uneven distribution of error across
|
|||
|
|
minorities or groups. In Diagram 8, Ádám Remport draws how different
|
|||
|
|
visual traits have often led to different error rates. While the general
|
|||
|
|
error metric of an algorithmic system might seem "acceptable," it
|
|||
|
|
actually privileges particular groups, which is invisible when only the
|
|||
|
|
whole is considered. Jeroen van Rest distinguishes such errors from the
|
|||
|
|
inherent algorithmic imprecision in deep machine learning models, as
|
|||
|
|
systemic biases (Diagram 7), as they perpetuate inequalities in the
|
|||
|
|
society in which the product is being developed.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Figure">
|
|||
|
|
<div class="anchor" data-i="2" style="height:4in"></div>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Caption">
|
|||
|
|
<p>Diagram 8. Ádám Remport describes that facial recognition
|
|||
|
|
technologies are often most accurate with white male adult faces,
|
|||
|
|
reflecting the datasets they are trained with. The FPR is higher with
|
|||
|
|
people with darker skin, children, or women, which may result in false
|
|||
|
|
flagging and false arrests.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>To mitigate these concerns and manage their risk, many of our
|
|||
|
|
interviewees who develop and implement these technologies, externalize
|
|||
|
|
the reference against which the error is measured. They turn to a
|
|||
|
|
benchmark run by the American National Institute of Standards and
|
|||
|
|
Technology (NIST), which ranks facial recognition technologies by
|
|||
|
|
different companies by their error metric across groups. John Riemen,
|
|||
|
|
who is responsible for the use of forensic facial recognition technology
|
|||
|
|
at the Center for Biometrics of the Dutch police, describes how their
|
|||
|
|
choice for software is driven by a public tender that demands a "top-10"
|
|||
|
|
score on the NIST benchmark. The mitigation of bias is thus outsourced
|
|||
|
|
to an external, and in this case foreign, institution.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>We see in this outsourcing of error metrics a form of delegation that
|
|||
|
|
brings about a specific regime of (in)visibility. While a particular
|
|||
|
|
kind of algorithmic bias is rendered central to the NIST benchmark, the
|
|||
|
|
mobilization of this reference obfuscates questions on how that metric
|
|||
|
|
was achieved. That is to say, questions about training data are
|
|||
|
|
invisibilized, even though that data is a known site of contestation.
|
|||
|
|
For example, the NIST benchmark datasets are known to include faces of
|
|||
|
|
wounded people (Keyes, 2019). The Clearview company is known to use
|
|||
|
|
images scraped illegally from social media, and IBM uses a dataset that
|
|||
|
|
is likely in violation of European GDPR legislation (Bommasani et al.,
|
|||
|
|
2022: 154). Pasquinelli (2019) argued that machine learning models
|
|||
|
|
ultimately act as data compressors: enfolding and operationalizing
|
|||
|
|
imagery of which the terms of acquisition are invisibilized.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>Attention to this invisibilization reveals a discrepancy between the
|
|||
|
|
developers and the implementers of these technologies. On the one hand,
|
|||
|
|
the developers we interviewed expressed concerns about how their
|
|||
|
|
training data is constituted to gain a maximum false positive rate/true
|
|||
|
|
positive rate (FPR/TPR) ratio, while showing concern for the legality of
|
|||
|
|
the data they use to train their algorithms. On the other hand,
|
|||
|
|
questions about the constitution of the dataset have been virtually
|
|||
|
|
non-existent in our conversations with those who implement software that
|
|||
|
|
relies on models trained with such data. Occasionally this knowledge was
|
|||
|
|
considered part of the developers' intellectual property that had to be
|
|||
|
|
kept a trade secret. A high score on the benchmark is enough to pass
|
|||
|
|
questions of fairness, legitimizing the use of the algorithmic model.
|
|||
|
|
Thus, while indirectly relying on the source data, it is no longer
|
|||
|
|
deemed relevant in the consideration of an algorithm. This illustrates
|
|||
|
|
well how the invisibilization of the “compressed” dataset, in
|
|||
|
|
Pasquinelli’s terms, into a model, with the formalization of guiding
|
|||
|
|
metrics into a benchmark, permits a bracketing of accountability. One
|
|||
|
|
does not need to know how outcomes are produced, as long as the
|
|||
|
|
benchmarks are in order.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>The configuration of algorithmic vision’s bias across a complex
|
|||
|
|
network of fragmented locations and actors, from the dataset, to the
|
|||
|
|
algorithm, to the benchmark institution reveals the selective processes
|
|||
|
|
of (in)visibilization. This opens up fruitful alleys for new empirical
|
|||
|
|
research: What are the politics of the benchmark as a mechanism of
|
|||
|
|
legitimization? How does the outsourcing of assessing the error
|
|||
|
|
distribution impact attention to bias? How has the critique of bias been
|
|||
|
|
institutionalized by the security industry, resulting in the
|
|||
|
|
externalization of accountability, through dis-location and
|
|||
|
|
fragmentation?</p>
|
|||
|
|
</div>
|
|||
|
|
</section>
|
|||
|
|
<section id="reconfiguring-the-human-in-the-loop" class="level2">
|
|||
|
|
<h2>Reconfiguring the human-in-the-loop</h2>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>A second central question linked to the delegation of accountability
|
|||
|
|
is the configuration in which the security operator is located. The
|
|||
|
|
effects of delegation and fragmentation in which the mitigation of
|
|||
|
|
algorithmic errors is outsourced to an external party, becomes visible
|
|||
|
|
in the ways in which the role of the security operator is configured in
|
|||
|
|
relation to the institution they work for, the software’s assessment,
|
|||
|
|
and the affected publics.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>The public critique of algorithms has often construed the
|
|||
|
|
<em>human-in-the loop</em> as one of the last lines of defense in the
|
|||
|
|
resistance to automated systems, able to filter and correct erroneous
|
|||
|
|
outcomes (Markoff, 2020). The literature in critical security studies
|
|||
|
|
has however problematized the representation of the security operator in
|
|||
|
|
algorithmic assemblages by discussing how the algorithmic predictions
|
|||
|
|
appear on their screen (Aradau and Blanke, 2018), and how the embodied
|
|||
|
|
decision making of the operator is entangled with the algorithmic
|
|||
|
|
assemblage (Wilcox, 2017). Moreover, the operator is often left guessing
|
|||
|
|
at the working of the device that provides them with information to make
|
|||
|
|
their decision (Møhl, 2021).
|
|||
|
|
</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>What our participants’ diagrams emphasized is how a whole spectrum of
|
|||
|
|
system designs emerges in response to similar questions, for example the
|
|||
|
|
issue of algorithmic bias. A primary difference can be found in the
|
|||
|
|
degree of understanding of the systems that is expected of security
|
|||
|
|
operators, as well as their perceived autonomy. Sometimes, the human
|
|||
|
|
operator is central to the system’s operation, forming the interface
|
|||
|
|
between the algorithmic systems and surveillance practices. Gerwin van
|
|||
|
|
der Lugt, developer of software at Oddity.ai that detects criminal
|
|||
|
|
behavior argues that “the responsibility for how to deal with the
|
|||
|
|
violent incidents is always [on a] human, not the algorithm. The
|
|||
|
|
algorithm just detects violence—that’s it—but the human needs to deal
|
|||
|
|
with it.”</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>Dirk Herzbach, chief of police at the Police Headquarters Mannheim,
|
|||
|
|
adds that when alerted to an incident by the system, the operator
|
|||
|
|
decides whether to deploy a police car. Both Herzbach and Van der Lugt
|
|||
|
|
figure the human-in-the-loop as having full agency and responsibility in
|
|||
|
|
operating the (in)security assemblage (cf. Hoijtink and Leese,
|
|||
|
|
2019).</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>Some interviewees drew a diagram in which the operator is supposed to
|
|||
|
|
be aware of the ways in which the technology errs, so they can address
|
|||
|
|
them. Several other interviewees considered the technical expertise of
|
|||
|
|
the human-in-the-loop to be unimportant, even a hindrance.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>Chief of police Herzbach prefers an operator to have patrol
|
|||
|
|
experience to assess which situations require intervention. He is
|
|||
|
|
concerned that knowledge about algorithmic biases might interfere with
|
|||
|
|
such decisions. In the case of the Moscow metro, in which a facial
|
|||
|
|
recognition system has been deployed to purchase tickets and open access
|
|||
|
|
gates, the human-in-the-loop is reconfigured as an end user who needs to
|
|||
|
|
be shielded from the algorithm’s operation (c.f. Lorusso, 2021). On
|
|||
|
|
these occasions, expertise on the technological creation of the suspect
|
|||
|
|
becomes fragmented.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>These different figurations of the security operator are held
|
|||
|
|
together by the idea that the human operator is the expert of the
|
|||
|
|
subject of security, and is expected to make decisions independent from
|
|||
|
|
the information that the algorithmic system provides.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Figure">
|
|||
|
|
<div class="anchor" data-i="3" style="height:6in"></div>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Caption">
|
|||
|
|
<p>Diagram 9. Riemen explains the process of information filtering that
|
|||
|
|
is involved in querying the facial recognition database of the Dutch
|
|||
|
|
police.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>Other drivers exist, however, to shield the operator from the
|
|||
|
|
algorithm’s functioning, challenging individual expertise and
|
|||
|
|
acknowledging the fallibility of human decision making. In Diagram 9,
|
|||
|
|
John Riemen outlines the use of facial recognition by the Dutch police.
|
|||
|
|
He describes how data from the police case and on the algorithmic
|
|||
|
|
assessment is filtered out as much as possible from the information
|
|||
|
|
provided to the operator. This, Riemen suggests, might reduce bias in
|
|||
|
|
the final decision. He adds that there should be no fewer than three
|
|||
|
|
humans-in-the-loop who operate independently to increase the accuracy of
|
|||
|
|
the algorithmic security vision.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>Instead of increasing their number, there is another configuration of
|
|||
|
|
the human-in-the-loop that responds to the fallibility of the operator.
|
|||
|
|
For the Burglary-Free Neighborhood project in Rotterdam, project manager
|
|||
|
|
Guido Delver draws surveillance as operated by neighborhood residents,
|
|||
|
|
through a system that they own themselves. By involving different
|
|||
|
|
stakeholders, Delver hopes to counter government hegemony over the
|
|||
|
|
surveillance apparatus. However, residents are untrained in assessing
|
|||
|
|
algorithmic predictions raising new challenges. Delver illustrates a
|
|||
|
|
scenario in which the algorithmic signaling of a potential burglary may
|
|||
|
|
have dangerous consequences: “Does it invoke the wrong behavior from the
|
|||
|
|
citizen? [They could] go out with a bat and look for the guy who has
|
|||
|
|
done nothing [because] it was a false positive.” In this case, the worry
|
|||
|
|
is that the erroneous predictions will not be questioned. Therefore, in
|
|||
|
|
Delver’s project the goal was to actualize an autonomous system, “with
|
|||
|
|
as little interference as possible.” Human participation or
|
|||
|
|
“interference” in the operation are potentially harmful. Thus, figuring
|
|||
|
|
the operator — whether police officer or neighborhood resident — as
|
|||
|
|
risky, can lead to the relegation of direct human intervention.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>By looking at the figurations of the operator that appear in the
|
|||
|
|
diagrams we see multiple and heterogeneous configurations of
|
|||
|
|
regulations, security companies, and professionals. In each
|
|||
|
|
configuration, the human-in-the-loop appears in different forms. The
|
|||
|
|
operator often holds the final responsibility in the ethical functioning
|
|||
|
|
of the system. At times they are configured as an expert in
|
|||
|
|
sophisticated but error-prone systems; at others they are figured as end
|
|||
|
|
users who are activated by the alerts generated by the system, and who
|
|||
|
|
need not understand how the software works and errs, or who can be left
|
|||
|
|
out.</p>
|
|||
|
|
</div>
|
|||
|
|
<div data-custom-style="Body Text">
|
|||
|
|
<p>These configurations remind us that there cannot be any theorization
|
|||
|
|
of “algorithmic security vision,” both of its empirical workings and its
|
|||
|
|
ethical and political consequences without close attention to the
|
|||
|
|
empirical contexts in which the configurations are arranged. Each
|
|||
|
|
organization of datasets, algorithms, benchmarks, hardware and operators
|
|||
|
|
has specific problems. And each contains specific politics of
|
|||
|
|
visibilization, invisibilization, responsibility and accountability.</p>
|
|||
|
|
</div>
|
|||
|
|
</section>
|
|||
|
|
</body>
|
|||
|
|
|
|||
|
|
</html>
|