lily
/
ipmap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add raw scan data functionality

This commit is contained in:
LilyRose2798 2024-04-22 08:35:12 +10:00
parent eebcbae626
commit b06a53c6c6
4 changed files with 172 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
ipmap.py
View File

@ -10,11 +10,11 @@ from shutil import rmtree
from gc import collect from gc import collect
from json import loads, dumps from json import loads, dumps
from pathlib import Path from pathlib import Path
from dataclasses import dataclass from typing import TypeVar
from typing import Literal, TypeVar
from png import Writer
from cmap import Colormap from cmap import Colormap
from hilbert import decode from hilbert import decode
from zlib import compress, crc32
from struct import pack
from numpy.typing import NDArray from numpy.typing import NDArray
import numpy as np import numpy as np
@ -133,38 +133,43 @@ def make_tiles(coords_path: Path, input_path: Path, tiles_dir: Path, *,
else: else:
tiles_dir_parts = None tiles_dir_parts = None
def create_tile_images(data: np.ndarray, colormap: Colormap, num_colors: int, path: Path): def get_chunk(tag: bytes, data = b""):
print(f"creating {num_colors} color stop(s) of {colormap.name} colormap...", end = " ", flush = True) return b"".join((pack("!I", len(data)), tag, data, pack("!I", crc32(data, crc32(tag)) & (2 ** 32 - 1))))
colors = np.concatenate(([empty_color], ((colormap([0.0]) if num_colors == 1 else colormap.lut(num_colors))[:, 0:channels] * 255).astype(np.uint8)))
print("done") signature = b"\x89PNG\r\n\x1a\n"
print(f"creating {data.shape[1]}x{data.shape[0]} pixel image for {colormap.name} colormap...", end = " ", flush = True) def get_preamble(alpha: bool):
image_data = colors[data] return signature + get_chunk(b"IHDR", pack("!2I5B", tile_size, tile_size, 8, 6 if alpha else 2, 0, 0, 0))
print("done") rgb_preamble = get_preamble(False)
del colors rgba_preamble = get_preamble(True)
collect() end_chunk = get_chunk(b"IEND")
tiles_per_side = image_data.shape[0] // tile_size
def create_tiles(path: Path, data: np.ndarray, colors: NDArray[np.uint8] | None = None):
tiles_per_side = data.shape[0] // tile_size
z = tiles_per_side.bit_length() - 1 z = tiles_per_side.bit_length() - 1
z_path = path / f"{z}" z_path = path / f"{z}"
z_path.mkdir(exist_ok = True, parents = True) z_path.mkdir(exist_ok = True, parents = True)
print(f"writing {tiles_per_side * tiles_per_side} ({tiles_per_side}x{tiles_per_side}) images to '{path}'...", end = " ", flush = True) def tile_generator():
for y in range(tiles_per_side): for y in range(tiles_per_side):
y_path = z_path / f"{y}" y_path = z_path / f"{y}"
y_path.mkdir(exist_ok = True) y_path.mkdir(exist_ok = True)
for x in range(tiles_per_side): for x in range(tiles_per_side):
x_path = y_path / f"{x}.png" yield (y_path, x, data[
rows = image_data[ y * tile_size : y * tile_size + tile_size,
y * tile_size : y * tile_size + tile_size, x * tile_size : x * tile_size + tile_size,
x * tile_size : x * tile_size + tile_size, ])
] print(f"writing {tiles_per_side * tiles_per_side} ({tiles_per_side}x{tiles_per_side}) tiles to '{z_path}'...", end = " ", flush = True)
Writer(tile_size, tile_size, greyscale = False, alpha = alpha).write_packed(x_path.open("wb"), rows) if colors is None:
for y_path, x, tile in tile_generator():
(y_path / f"{x}.bin").write_bytes(compress(tile.tobytes()))
else:
preamble = rgb_preamble if colors.shape[1] == 3 else rgba_preamble
for y_path, x, tile in tile_generator():
idat_chunk = get_chunk(b"IDAT", compress(np.insert(colors[tile].reshape(tile_size, -1), 0, 0, axis = 1).tobytes()))
(y_path / f"{x}.png").write_bytes(b"".join((preamble, idat_chunk, end_chunk)))
print("done") print("done")
def create_raw_image(data: np.ndarray, path: Path): def get_colors(colormap: Colormap, num_colors: int):
path.mkdir(exist_ok = True, parents = True) return np.concatenate(([empty_color], ((colormap([0.0]) if num_colors == 1 else colormap.lut(num_colors))[:, 0:channels] * 255).astype(np.uint8)))
z_path = path / f"{(data.shape[0] // tile_size).bit_length() - 1}.png"
print(f"writing {data.shape[1]}x{data.shape[0]} raw image to '{path}'...", end = " ", flush = True)
Writer(data.shape[1], data.shape[0], greyscale = False, alpha = True).write_packed(z_path.open("wb"), data)
print("done")
def get_scan_data() -> tuple[NDArray[np.uint32], NDArray[np.uint32]]: def get_scan_data() -> tuple[NDArray[np.uint32], NDArray[np.uint32]]:
print(f"reading scan data from file '{input_path}'...", end = " ", flush = True) print(f"reading scan data from file '{input_path}'...", end = " ", flush = True)
@ -205,8 +210,10 @@ def make_tiles(coords_path: Path, input_path: Path, tiles_dir: Path, *,
density_data[:, :, 0, 0] += density_data[:, :, 0, 1] density_data[:, :, 0, 0] += density_data[:, :, 0, 1]
density_data[:, :, 0, 0] += density_data[:, :, 1, 0] density_data[:, :, 0, 0] += density_data[:, :, 1, 0]
density_data[:, :, 0, 0] += density_data[:, :, 1, 1] density_data[:, :, 0, 0] += density_data[:, :, 1, 1]
print(f"done (shrunk density data from {density_data.shape[0] * 2}x{density_data.shape[1] * 2} -> {density_data.shape[0]}x{density_data.shape[1]})") print("done")
density_data = density_data[:, :, 0, 0] print(f"shrinking density data from {density_data.shape[0]}x{density_data.shape[1]} to {density_data.shape[0] // 2}x{density_data.shape[1] // 2}...", end = " ", flush = True)
density_data = np.copy(density_data[:, :, 0, 0])
print("done")
possible_overlaps *= 4 possible_overlaps *= 4
if skip_iters is not None: if skip_iters is not None:
@ -214,10 +221,10 @@ def make_tiles(coords_path: Path, input_path: Path, tiles_dir: Path, *,
squish() squish()
def write_all_colormaps(): def write_all_colormaps():
for colormap_name, colormap in colormaps:
create_tile_images(density_data, colormap, possible_overlaps, tiles_dir / variant_name / colormap_name)
if raws_path is not None: if raws_path is not None:
create_raw_image(density_data, raws_path / variant_name) create_tiles(raws_path / variant_name, density_data.view(np.uint8).reshape(density_data.shape[0], density_data.shape[1], 4))
for colormap_name, colormap in colormaps:
create_tiles(tiles_dir / variant_name / colormap_name, density_data, get_colors(colormap, possible_overlaps))
write_all_colormaps() write_all_colormaps()
while density_data.shape[0] > tile_size: while density_data.shape[0] > tile_size:
@ -276,14 +283,16 @@ def make_tiles(coords_path: Path, input_path: Path, tiles_dir: Path, *,
rtt_data[mask, 0, 1] //= 2 rtt_data[mask, 0, 1] //= 2
rtt_data[mask, 0, 0] += rtt_data[mask, 0, 1] # take average of first two nums rtt_data[mask, 0, 0] += rtt_data[mask, 0, 1] # take average of first two nums
# everything else (1 or 0 nums populated) don't need any modifications # everything else (1 or 0 nums populated) don't need any modifications
print(f"done (shrunk rtt data from {rtt_data.shape[0] * 2}x{rtt_data.shape[1] * 2} -> {rtt_data.shape[0]}x{rtt_data.shape[1]})") print("done")
rtt_data = rtt_data[:, :, 0, 0] print(f"shrinking rtt data from {rtt_data.shape[0]}x{rtt_data.shape[1]} to {rtt_data.shape[0] // 2}x{rtt_data.shape[1] // 2}...", end = " ", flush = True)
rtt_data = np.copy(rtt_data[:, :, 0, 0])
print("done")
if skip_iters is not None: if skip_iters is not None:
for _ in range(skip_iters): for _ in range(skip_iters):
squish() squish()
def get_normalized_data(): def normalize():
print("normalizing rtt data: getting non-zero...", end = " ", flush = True) print("normalizing rtt data: getting non-zero...", end = " ", flush = True)
non_zero = rtt_data != 0 non_zero = rtt_data != 0
print("converting to floating point...", end = " ", flush = True) print("converting to floating point...", end = " ", flush = True)
@ -304,10 +313,10 @@ def make_tiles(coords_path: Path, input_path: Path, tiles_dir: Path, *,
def write_all_colormaps(): def write_all_colormaps():
if raws_path is not None: if raws_path is not None:
create_raw_image(rtt_data, raws_path / variant_name) create_tiles(raws_path / variant_name, rtt_data.view(np.uint8).reshape(rtt_data.shape[0], rtt_data.shape[1], 4))
rtt_data_norm = get_normalized_data() rtt_data_norm = normalize()
for colormap_name, colormap in colormaps: for colormap_name, colormap in colormaps:
create_tile_images(rtt_data_norm, colormap, num_colors, tiles_dir / variant_name / colormap_name) create_tiles(tiles_dir / variant_name / colormap_name, rtt_data_norm, get_colors(colormap, num_colors))
write_all_colormaps() write_all_colormaps()
while rtt_data.shape[0] > tile_size: while rtt_data.shape[0] > tile_size:

13
poetry.lock generated
View File

@ -75,18 +75,7 @@ files = [
{file = "numpy-hilbert-curve-1.0.1.tar.gz", hash = "sha256:0745dbd4c16b258c180342d6df57dfa99110b9d98c86a84d920f29af5cc0707b"}, {file = "numpy-hilbert-curve-1.0.1.tar.gz", hash = "sha256:0745dbd4c16b258c180342d6df57dfa99110b9d98c86a84d920f29af5cc0707b"},
] ]
[[package]]
name = "pypng"
version = "0.20220715.0"
description = "Pure Python library for saving and loading PNG images"
optional = false
python-versions = "*"
files = [
{file = "pypng-0.20220715.0-py3-none-any.whl", hash = "sha256:4a43e969b8f5aaafb2a415536c1a8ec7e341cd6a3f957fd5b5f32a4cfeed902c"},
{file = "pypng-0.20220715.0.tar.gz", hash = "sha256:739c433ba96f078315de54c0db975aee537cbc3e1d0ae4ed9aab0ca1e427e2c1"},
]
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = "^3.11" python-versions = "^3.11"
content-hash = "6f18c3faab65fe3440461e20f9200f5665981d0ba9d69f1dc8a1740840108ab1" content-hash = "7b7fb0cb9bc597ae838486c3f91be6d24679db0784b7d2813f1826bb305e9279"

209
public/index.html
View File

@ -73,6 +73,7 @@
font-size: 0.9rem; font-size: 0.9rem;
padding: 0.3rem 2.2rem 0.3rem 0.6rem; padding: 0.3rem 2.2rem 0.3rem 0.6rem;
border-radius: 4px; border-radius: 4px;
white-space: nowrap;
} }
.maplibregl-popup-close-button { .maplibregl-popup-close-button {
height: 100%; height: 100%;
@ -546,108 +547,138 @@
dateControl.addControl() dateControl.addControl()
variantControl.addControl() variantControl.addControl()
colormapControl.addControl() colormapControl.addControl()
})
map.addControl(new maplibregl.NavigationControl({ showCompass: false }), "top-left") map.addControl(new maplibregl.NavigationControl({ showCompass: false }), "top-left")
const hoverTextControl = document.createElement("div") const hoverTextControl = document.createElement("div")
hoverTextControl.className = "maplibregl-ctrl maplibregl-ctrl-group" hoverTextControl.className = "maplibregl-ctrl maplibregl-ctrl-group"
const hoverTextP = document.createElement("p") const hoverTextP = document.createElement("p")
hoverTextControl.replaceChildren(hoverTextP) hoverTextControl.replaceChildren(hoverTextP)
const ipFromMouseEvent = e => { const ipFromMouseEvent = e => {
const { x, y } = maplibregl.MercatorCoordinate.fromLngLat(e.lngLat, 0) const { x, y } = maplibregl.MercatorCoordinate.fromLngLat(e.lngLat, 0)
const ip = coordsToHilbert({ x: Math.floor(0x10000 * x), y: Math.floor(0x10000 * y) }) const ip = coordsToHilbert({ x: Math.floor(0x10000 * x), y: Math.floor(0x10000 * y) })
const subnet = Math.min(32, Math.round(map.getZoom()) * 2 + 18) const subnet = Math.min(32, Math.round(map.getZoom()) * 2 + 18)
return { ip, subnet } return { ip, subnet }
}
map.on("mousemove", e => {
const { ip, subnet } = ipFromMouseEvent(e)
const ipStr = ipToString(ip, subnet)
hoverTextP.textContent = subnet < 32 ? `Range: ${ipStr}/${subnet}` : `IP: ${ipStr}`
})
map.addControl({
onAdd: () => hoverTextControl,
onRemove: () => hoverTextControl.parentNode.removeChild(hoverTextControl)
}, "bottom-left")
let curPopup
const setPopup = (pos, ip, subnet = 32) => {
const isRange = subnet < 32
const name = isRange ? "Range" : "IP"
const ipStr = ipToString(ip, subnet)
const ipText = isRange ? `${ipStr}/${subnet}` : ipStr
const ipLink = `<a href="https://bgp.tools/prefix/${ipStr}" target="_blank">${ipText}</a>`
const htmlBase = `${name}: ${ipLink}`
const privateRange = getPrivateRange(ip)
const html = privateRange ? `${htmlBase}<br>Part of private range ${privateRange.range}<br>Used for ${privateRange.description}` : htmlBase
curPopup?.remove()
const popup = new maplibregl.Popup({ focusAfterOpen: false }).setHTML(html).setLngLat(pos).addTo(map)
curPopup = popup
if (!isRange && !privateRange) {
fetch(`${apiUrl}/api/rdns/${ipStr}`).then(res => {
if (!res.ok)
throw new Error(`Error fetching rdns for ip ${ipStr}`)
return res.json()
}).then(data => {
const rdns = data?.rdns
if (rdns) popup.setHTML(`${html}<br>rDNS: ${rdns}`)
}).catch(_ => {})
} }
}
map.on("click", e => { map.on("mousemove", e => {
const { ip, subnet } = ipFromMouseEvent(e) const { ip, subnet } = ipFromMouseEvent(e)
setPopup(e.lngLat, ip, subnet) const ipStr = ipToString(ip, subnet)
}) hoverTextP.textContent = subnet < 32 ? `Range: ${ipStr}/${subnet}` : `IP: ${ipStr}`
})
const main = document.getElementsByTagName("main")[0] map.addControl({
main.addEventListener("click", e => { onAdd: () => hoverTextControl,
if (e.target === main && curPopup) curPopup.remove() onRemove: () => hoverTextControl.parentNode.removeChild(hoverTextControl)
}) }, "bottom-left")
const rangeInput = document.getElementById("range-input")
const jumpParam = new URLSearchParams(location.search).get("jump")
const jump = range => { let curPopup
const { ip, subnet } = ipFromString(range) const setPopup = (pos, ip, subnet = 32) => {
const { x, y } = hilbertToCoords(ip + Math.floor((2 ** (32 - subnet)) / 2)) const isRange = subnet < 32
const center = new maplibregl.MercatorCoordinate((x + 0.5) / 0x10000, (y + 0.5) / 0x10000, 0).toLngLat() const name = isRange ? "Range" : "IP"
const zoom = subnet / 2 - 0.501 const ipStr = ipToString(ip, subnet)
map.jumpTo({ center, zoom }) const ipText = isRange ? `${ipStr}/${subnet}` : ipStr
if (subnet > 24) setPopup(center, ip, subnet) const ipLink = `<a href="https://bgp.tools/prefix/${ipStr}" target="_blank">${ipText}</a>`
} const htmlBase = `${name}: ${ipLink}`
const privateRange = getPrivateRange(ip)
let html = privateRange ? `${htmlBase}<br>Part of private range ${privateRange.range}<br>Used for ${privateRange.description}` : htmlBase
const setJumpParam = jump => { curPopup?.remove()
const searchParams = new URLSearchParams(location.search) const popup = new maplibregl.Popup({ focusAfterOpen: false }).setHTML(html).setLngLat(pos).addTo(map)
if (jump) searchParams.set("jump", jump) curPopup = popup
else searchParams.delete("jump")
history.pushState("", "", searchParams.size === 0 ? location.pathname : `?${searchParams}`)
}
try { if (!privateRange) {
jump(jumpParam) fetch(`${apiUrl}/api/scandata/${curDate}/rtt/range/${encodeURIComponent(`${ipStr}/${subnet}`)}`).then(res => {
rangeInput.value = jumpParam if (!res.ok)
} catch(e) { throw new Error(`Error fetching scan data for range ${ipStr}`)
setJumpParam(undefined) return res.json()
} }).then(data => {
const rtt = data?.rtt
if (rtt) {
html = `${html}<br>RTT: ${(rtt / 1000).toFixed(2)}ms`
popup.setHTML(html)
}
}).catch(_ => {})
if (isRange)
fetch(`${apiUrl}/api/scandata/${curDate}/density/range/${encodeURIComponent(`${ipStr}/${subnet}`)}`).then(res => {
if (!res.ok)
throw new Error(`Error fetching scan data for range ${ipStr}`)
return res.json()
}).then(data => {
const density = data?.density
if (density !== undefined) {
const possibleOverlaps = 2 ** (32 - subnet)
const densityPct = (100 * (density / possibleOverlaps)).toFixed(2)
html = `${html}<br>Density: ${densityPct}% (${density}/${possibleOverlaps})`
popup.setHTML(html)
}
}).catch(_ => {})
else
fetch(`${apiUrl}/api/rdns/${encodeURIComponent(ipStr)}`).then(res => {
if (!res.ok)
throw new Error(`Error fetching rdns for ip ${ipStr}`)
return res.json()
}).then(data => {
const rdns = data?.rdns
if (rdns) {
html = `${html}<br>rDNS: ${rdns}`
popup.setHTML(html)
}
}).catch(_ => {})
}
}
map.on("click", e => {
const { ip, subnet } = ipFromMouseEvent(e)
setPopup(e.lngLat, ip, subnet)
})
const main = document.getElementsByTagName("main")[0]
main.addEventListener("click", e => {
if (e.target === main && curPopup) curPopup.remove()
})
const rangeInput = document.getElementById("range-input")
const jumpParam = new URLSearchParams(location.search).get("jump")
const jump = range => {
const { ip, subnet } = ipFromString(range)
const { x, y } = hilbertToCoords(ip + Math.floor((2 ** (32 - subnet)) / 2))
const center = new maplibregl.MercatorCoordinate((x + 0.5) / 0x10000, (y + 0.5) / 0x10000, 0).toLngLat()
const zoom = subnet / 2 - 0.501
map.jumpTo({ center, zoom })
if (subnet > 24) setPopup(center, ip, subnet)
}
const setJumpParam = jump => {
const searchParams = new URLSearchParams(location.search)
if (jump) searchParams.set("jump", jump)
else searchParams.delete("jump")
history.pushState("", "", searchParams.size === 0 ? location.pathname : `?${searchParams}`)
}
const jumpToInput = () => {
try { try {
if (rangeInput.value) jump(rangeInput.value) jump(jumpParam)
setJumpParam(rangeInput.value) rangeInput.value = jumpParam
rangeInput.setCustomValidity("") } catch(e) {
} catch (e) { setJumpParam(undefined)
rangeInput.setCustomValidity(e.message)
} }
rangeInput.reportValidity()
}
rangeInput.addEventListener("change", jumpToInput) const jumpToInput = () => {
document.getElementById("range-button").addEventListener("click", jumpToInput) try {
if (rangeInput.value) jump(rangeInput.value)
setJumpParam(rangeInput.value)
rangeInput.setCustomValidity("")
} catch (e) {
rangeInput.setCustomValidity(e.message)
}
rangeInput.reportValidity()
}
rangeInput.addEventListener("change", jumpToInput)
document.getElementById("range-button").addEventListener("click", jumpToInput)
})
</script> </script>
</body> </body>
</html> </html>

1
pyproject.toml
View File

@ -7,7 +7,6 @@ license = "AGPLv3"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.11" python = "^3.11"
pypng = "^0.20220715.0"
numpy = "^1.26.4" numpy = "^1.26.4"
numpy-hilbert-curve = "^1.0.1" numpy-hilbert-curve = "^1.0.1"
cmap = "^0.1.3" cmap = "^0.1.3"