Fixed it!
Private Function ElementHit(Element As PersonViewModel, x As Integer, y As Integer) As Boolean
Return Abs(x - Element.Left * ZoomFactor) < Element.Width * ZoomFactor And _
Abs(y - Element.Top * ZoomFactor) < Element.Height * ZoomFactor And _
x > Element.Left * ZoomFactor And _
y > Element.Top * ZoomFactor
End Function