lily
/
ipmap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit

This commit is contained in:
LilyRose2798 2024-04-01 11:21:11 +11:00
commit 5a519bd80e
6 changed files with 400 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
data/
tiles/

1
README.md Normal file
View File

@ -0,0 +1 @@
# IP Map

149
index.html Normal file
View File

@ -0,0 +1,149 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="color-scheme" content="dark light">
<title>IP Map</title>
<script src="https://unpkg.com/maplibre-gl@4.1.2/dist/maplibre-gl.js"></script>
<link rel="stylesheet" href="https://unpkg.com/maplibre-gl@4.1.2/dist/maplibre-gl.css" />
<style>
body {
margin: 0;
padding: 0;
}
html, body {
width: 100%;
height: 100%;
}
#map {
width: 100vh;
height: 100%;
margin: 0 auto;
}
.maplibregl-canvas {
cursor: pointer;
}
.maplibregl-popup-content {
background-color: #222;
font-size: 1rem;
padding: 0.8rem 1.2rem;
}
</style>
</head>
<body>
<div id="map"></div>
<script>
function hilbert_c2i({ x, y }) {
var b, d;
var rotation = 0;
let reflection = 0;
let index = 0;
for (b = 16; b--;) {
let bits = reflection;
reflection = (y >>> b) & 1;
reflection |= ((x >>> b) & 1) << 1;
bits = bits ^ reflection;
bits = ((bits >>> rotation) | (bits << (2 - rotation))) & 3;
index |= bits << (b << 1);
reflection ^= (1 << rotation);
bits = bits & (-bits) & 1;
while (bits) { ++rotation; bits >>>= 1; }
if (++rotation >= 2) rotation -= 2;
}
index ^= 0x2aaaaaaa;
for (d = 1; d < 32; d *= 2) {
let t = index >>> d;
if (!t) break;
index ^= t;
}
return index;
}
function hilbert_i2c(index) {
var b, d, t;
var rotation = 0;
let reflection = 0;
let coord = { x: 0, y: 0 };
index ^= (index >>> 1) ^ 0x2aaaaaaa;
for (b = 16; b--; ) {
var bits = index >>> (2 * b) & 3;
reflection ^= ((bits >>> (2 - rotation)) | (bits << rotation)) & 3;
coord.x |= (reflection & 1) << b;
coord.y |= ((reflection >>> 1) & 1) << b;
reflection ^= (1 << rotation);
bits &= (-bits) & 1;
while (bits) {
bits >>>= 1;
++rotation;
}
if (++rotation >= 2)
rotation -= 2;
}
return coord;
}
const map = new maplibregl.Map({
container: "map",
attributionControl: false,
renderWorldCopies: false,
doubleClickZoom: false,
dragRotate: false,
style: {
"version": 8,
"sources": {
"raster-tiles": {
"type": "raster",
"tiles": [
"tiles/{z}/{y}/{x}.png"
],
"minzoom": 0,
"maxzoom": 8,
"tileSize": 256
}
},
"layers": [
{
"id": "simple-tiles",
"type": "raster",
"source": "raster-tiles",
"paint": {
"raster-resampling": "nearest"
}
}
]
},
center: [0, 0],
minZoom: -1,
maxZoom: 12,
zoom: 0
})
map.painter.context.extTextureFilterAnisotropic = undefined
map.addControl(new maplibregl.NavigationControl({ showCompass: false }), "top-left")
const toIp = v => `${v >> 24 & 0xFF}.${v >> 16 & 0xFF}.${v >> 8 & 0xFF}.${v >> 0 & 0xFF}`
map.on("click", (e) => {
const { x, y } = maplibregl.MercatorCoordinate.fromLngLat(e.lngLat, 0)
const rawIp = hilbert_c2i({ x: Math.floor(0x10000 * x), y: Math.floor(0x10000 * y) })
const subnet = Math.min(32, Math.round(map.getZoom()) * 2 + 18)
const text = subnet < 32 ?
`Range: ${toIp((rawIp >> (32 - subnet)) << (32 - subnet))}/${subnet}` :
`IP: ${toIp(rawIp)}`
new maplibregl.Popup()
.setHTML(text)
.setLngLat(e.lngLat)
.addTo(map)
})
</script>
</body>
</html>

104
ipmap.py Normal file
View File

@ -0,0 +1,104 @@
import math
import functools
from pathlib import Path
import png
import hilbert
import numpy as np
import polars as pl
from multiprocessing import Pool
hilbert_coords = functools.partial(hilbert.decode, num_dims=2, num_bits=16)
def convert_to_parquet(csv_path: Path, parquet_path: Path):
if not csv_path.exists():
print(f"no csv found at \"{csv_path}\"")
return
lf = pl.scan_csv(csv_path, schema={
"saddr": pl.String,
"rtt_us": pl.UInt64,
# "success": pl.UInt8
})
# lf = lf.filter(pl.col("success") == 1)
# lf = lf.drop("success")
lf = lf.with_columns(rtt_us=pl.col("rtt_us").clip(0, 0xFFFFFFFF).cast(pl.UInt32))
lf = lf.with_columns(saddr=pl.col("saddr").str.split_exact(".", 3).struct.rename_fields(["a", "b", "c", "d"]))
lf = lf.with_columns(saddr=pl.col("saddr").struct.field("a").cast(pl.UInt32) * 0x1000000 + pl.col("saddr").struct.field("b").cast(pl.UInt32) * 0x10000 + pl.col("saddr").struct.field("c").cast(pl.UInt32) * 0x100 + pl.col("saddr").struct.field("d").cast(pl.UInt32))
lf = lf.with_columns(coords=pl.col("saddr").map_batches(hilbert_coords, pl.Array(pl.UInt16, 2), is_elementwise=True))
lf = lf.with_columns(x=pl.col("coords").arr.get(0), y=pl.col("coords").arr.get(1))
lf = lf.drop("coords")
print(f"scanning csv \"{csv_path}\" into parquet \"{parquet_path}\"...", end=" ", flush=True)
lf.sink_parquet(parquet_path)
print("done.")
def write_tile(path: Path, rows: np.ndarray):
path.parent.mkdir(exist_ok=True, parents=True)
png.Writer(rows.shape[0], rows.shape[0], greyscale=False, alpha=False).write(path.open("wb"), rows)
def generate_tiles(parquet_path: Path, tiles_dir: Path, tile_size=256, processes=16):
print(f"reading parquet \"{parquet_path}\"...", end=" ", flush=True)
gdf = pl.read_parquet(parquet_path, columns=["x", "y", "rtt_us"])
print("done.")
channels = 3
tile_size_by_channels = tile_size * channels
tiles_per_side = int(math.sqrt(0x100000000)) // tile_size
while True:
# if tiles_per_side <= 16:
z = int(math.log2(tiles_per_side))
print(f"[{z=}] calculating colors...", end=" ", flush=True)
df = gdf.with_columns(gb=0xFF - (pl.col("rtt_us") / 3000).round().clip(0, 0xFF).cast(pl.UInt8))
df = df.drop("rtt_us")
df = df.with_columns(r=0xFF, g=pl.col("gb"), b=pl.col("gb"))
df = df.drop("gb")
print("done.")
total_size = tiles_per_side * tile_size
print(f"[{z=}] creating image row data...", end=" ", flush=True)
all_rows = np.zeros((total_size, total_size, channels), dtype = "uint8")
all_rows[(df.get_column("y"), df.get_column("x"))] = df.select(["r", "g", "b"]).to_numpy()
all_rows = all_rows.reshape(total_size, total_size * channels)
print("done.")
del df
z_path = tiles_dir / f"{z}"
z_path.mkdir(exist_ok=True, parents=True)
print(f"[{z=}] creating individual tile data...", end=" ", flush=True)
tile_data = [
(z_path / f"{y}" / f"{x}.png", all_rows[
y * tile_size : y * tile_size + tile_size,
x * tile_size_by_channels : x * tile_size_by_channels + tile_size_by_channels
])
for x in range(tiles_per_side)
for y in range(tiles_per_side)
]
print("done.")
print(f"[{z=}] writing {tiles_per_side}x{tiles_per_side}={tiles_per_side * tiles_per_side} images...", end=" ", flush=True)
with Pool(processes) as pool:
pool.starmap(write_tile, tile_data)
print("done.")
del tile_data
del all_rows
if tiles_per_side == 1:
break
old_tiles_per_side = tiles_per_side
tiles_per_side //= 2
print(f"[{z=}] rescaling {len(gdf)} coords from {old_tiles_per_side}x{old_tiles_per_side} to {tiles_per_side}x{tiles_per_side} tiles...", end=" ", flush=True)
new_gdf = gdf.with_columns(x=pl.col("x") // 2, y=pl.col("y") // 2)
del gdf
gdf = new_gdf.group_by(["x", "y"]).mean()
print(f"done. {len(gdf)} coords remaining.")
def main():
data_dir = Path("data")
csv_path = data_dir / "full-scan.csv"
parquet_path = data_dir / "full-scan.parquet"
if not parquet_path.exists():
print(f"no parquet file found at \"{parquet_path}\", generating now...")
convert_to_parquet(csv_path, parquet_path)
tiles_dir = Path("tiles")
generate_tiles(parquet_path, tiles_dir)
if __name__ == "__main__":
main()

125
poetry.lock generated Normal file
View File

@ -0,0 +1,125 @@
# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
[[package]]
name = "iptools"
version = "0.7.0"
description = "Python utilites for manipulating IPv4 and IPv6 addresses"
optional = false
python-versions = "*"
files = [
{file = "iptools-0.7.0-py2.py3-none-any.whl", hash = "sha256:a91fc7478fd795ac6b2d47c869fb46db7666ffec817bcb0560ef119e204237f0"},
{file = "iptools-0.7.0.tar.gz", hash = "sha256:118a4f638bb5fa0123df56fe3be703b112a689167539bcc194f8698ccdd9e2ea"},
]
[package.extras]
testing = ["nose (>=1.0)"]
[[package]]
name = "numpy"
version = "1.26.4"
description = "Fundamental package for array computing in Python"
optional = false
python-versions = ">=3.9"
files = [
{file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"},
{file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"},
{file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"},
{file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"},
{file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"},
{file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"},
{file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"},
{file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"},
{file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"},
{file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"},
{file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"},
{file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"},
{file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"},
{file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"},
{file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"},
{file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"},
{file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"},
{file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"},
{file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"},
{file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"},
{file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"},
{file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"},
{file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"},
{file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"},
{file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"},
{file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"},
{file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"},
{file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"},
{file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"},
{file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"},
{file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"},
{file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"},
{file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"},
{file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"},
{file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"},
{file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"},
]
[[package]]
name = "numpy-hilbert-curve"
version = "1.0.1"
description = "Implements Hilbert space-filling curves for Python with numpy"
optional = false
python-versions = "*"
files = [
{file = "numpy-hilbert-curve-1.0.1.tar.gz", hash = "sha256:0745dbd4c16b258c180342d6df57dfa99110b9d98c86a84d920f29af5cc0707b"},
]
[[package]]
name = "polars-lts-cpu"
version = "0.20.17"
description = "Blazingly fast DataFrame library"
optional = false
python-versions = ">=3.8"
files = [
{file = "polars_lts_cpu-0.20.17-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c5ba1113df88bd0e46bc2e649279f1e2f09f20d24a7e3a8b07d342d1e117bf40"},
{file = "polars_lts_cpu-0.20.17-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:514e833c63d2734d9028ca754fe441479cb8d68d06efe9f88fdb348db9578941"},
{file = "polars_lts_cpu-0.20.17-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3512862da0bcb764ed5e63bb122d265295d503e5294c839d5f46f88937543cc1"},
{file = "polars_lts_cpu-0.20.17-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:2a30789e25a07e0c925e6fde030d2ee53024ae621a0194c423ff83f359d5f62c"},
{file = "polars_lts_cpu-0.20.17-cp38-abi3-win_amd64.whl", hash = "sha256:b5a3487d481517525d7c9b9c69210f123c2d1f233c47487fa058646c2dc3d42c"},
{file = "polars_lts_cpu-0.20.17.tar.gz", hash = "sha256:e11eb08f9264459339af4942c4be9c187daf2ffe4040d24284582e4e0e492ab7"},
]
[package.extras]
adbc = ["adbc-driver-manager", "adbc-driver-sqlite"]
all = ["polars[adbc,async,cloudpickle,connectorx,deltalake,fastexcel,fsspec,gevent,numpy,pandas,plot,pyarrow,pydantic,pyiceberg,sqlalchemy,timezone,xlsx2csv,xlsxwriter]"]
async = ["nest-asyncio"]
cloudpickle = ["cloudpickle"]
connectorx = ["connectorx (>=0.3.2)"]
deltalake = ["deltalake (>=0.14.0)"]
fastexcel = ["fastexcel (>=0.9)"]
fsspec = ["fsspec"]
gevent = ["gevent"]
matplotlib = ["matplotlib"]
numpy = ["numpy (>=1.16.0)"]
openpyxl = ["openpyxl (>=3.0.0)"]
pandas = ["pandas", "pyarrow (>=7.0.0)"]
plot = ["hvplot (>=0.9.1)"]
pyarrow = ["pyarrow (>=7.0.0)"]
pydantic = ["pydantic"]
pyiceberg = ["pyiceberg (>=0.5.0)"]
pyxlsb = ["pyxlsb (>=1.0)"]
sqlalchemy = ["pandas", "sqlalchemy"]
timezone = ["backports-zoneinfo", "tzdata"]
xlsx2csv = ["xlsx2csv (>=0.8.0)"]
xlsxwriter = ["xlsxwriter"]
[[package]]
name = "pypng"
version = "0.20220715.0"
description = "Pure Python library for saving and loading PNG images"
optional = false
python-versions = "*"
files = [
{file = "pypng-0.20220715.0-py3-none-any.whl", hash = "sha256:4a43e969b8f5aaafb2a415536c1a8ec7e341cd6a3f957fd5b5f32a4cfeed902c"},
{file = "pypng-0.20220715.0.tar.gz", hash = "sha256:739c433ba96f078315de54c0db975aee537cbc3e1d0ae4ed9aab0ca1e427e2c1"},
]
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "ae4649713b0931b2549ea9b0951c28123e2c1d7abcdf26bfd202ac2aa039d9db"

19
pyproject.toml Normal file
View File

@ -0,0 +1,19 @@
[tool.poetry]
name = "ipmap"
version = "0.1.0"
description = ""
authors = ["LilyRose2798 <lily@rose.place>"]
license = "AGPLv3"
[tool.poetry.dependencies]
python = "^3.11"
pypng = "^0.20220715.0"
iptools = "^0.7.0"
numpy = "^1.26.4"
numpy-hilbert-curve = "^1.0.1"
polars-lts-cpu = "^0.20.17"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"