From 5a519bd80e2ac61dc8794e27d638821317bb39ee Mon Sep 17 00:00:00 2001 From: LilyRose2798 Date: Mon, 1 Apr 2024 11:21:11 +1100 Subject: [PATCH] Initial commit --- .gitignore | 2 + README.md | 1 + index.html | 149 +++++++++++++++++++++++++++++++++++++++++++++++++ ipmap.py | 104 ++++++++++++++++++++++++++++++++++ poetry.lock | 125 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 19 +++++++ 6 files changed, 400 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 index.html create mode 100644 ipmap.py create mode 100644 poetry.lock create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dcf0d47 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +data/ +tiles/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..cac67e3 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# IP Map diff --git a/index.html b/index.html new file mode 100644 index 0000000..307da99 --- /dev/null +++ b/index.html @@ -0,0 +1,149 @@ + + + + + + + IP Map + + + + + +
+ + + diff --git a/ipmap.py b/ipmap.py new file mode 100644 index 0000000..67ac315 --- /dev/null +++ b/ipmap.py @@ -0,0 +1,104 @@ +import math +import functools +from pathlib import Path +import png +import hilbert +import numpy as np +import polars as pl +from multiprocessing import Pool + +hilbert_coords = functools.partial(hilbert.decode, num_dims=2, num_bits=16) + +def convert_to_parquet(csv_path: Path, parquet_path: Path): + if not csv_path.exists(): + print(f"no csv found at \"{csv_path}\"") + return + lf = pl.scan_csv(csv_path, schema={ + "saddr": pl.String, + "rtt_us": pl.UInt64, + # "success": pl.UInt8 + }) + # lf = lf.filter(pl.col("success") == 1) + # lf = lf.drop("success") + lf = lf.with_columns(rtt_us=pl.col("rtt_us").clip(0, 0xFFFFFFFF).cast(pl.UInt32)) + lf = lf.with_columns(saddr=pl.col("saddr").str.split_exact(".", 3).struct.rename_fields(["a", "b", "c", "d"])) + lf = lf.with_columns(saddr=pl.col("saddr").struct.field("a").cast(pl.UInt32) * 0x1000000 + pl.col("saddr").struct.field("b").cast(pl.UInt32) * 0x10000 + pl.col("saddr").struct.field("c").cast(pl.UInt32) * 0x100 + pl.col("saddr").struct.field("d").cast(pl.UInt32)) + lf = lf.with_columns(coords=pl.col("saddr").map_batches(hilbert_coords, pl.Array(pl.UInt16, 2), is_elementwise=True)) + lf = lf.with_columns(x=pl.col("coords").arr.get(0), y=pl.col("coords").arr.get(1)) + lf = lf.drop("coords") + print(f"scanning csv \"{csv_path}\" into parquet \"{parquet_path}\"...", end=" ", flush=True) + lf.sink_parquet(parquet_path) + print("done.") + +def write_tile(path: Path, rows: np.ndarray): + path.parent.mkdir(exist_ok=True, parents=True) + png.Writer(rows.shape[0], rows.shape[0], greyscale=False, alpha=False).write(path.open("wb"), rows) + +def generate_tiles(parquet_path: Path, tiles_dir: Path, tile_size=256, processes=16): + print(f"reading parquet \"{parquet_path}\"...", end=" ", flush=True) + gdf = pl.read_parquet(parquet_path, columns=["x", "y", "rtt_us"]) + print("done.") + + channels = 3 + tile_size_by_channels = tile_size * channels + tiles_per_side = int(math.sqrt(0x100000000)) // tile_size + + while True: + # if tiles_per_side <= 16: + z = int(math.log2(tiles_per_side)) + print(f"[{z=}] calculating colors...", end=" ", flush=True) + df = gdf.with_columns(gb=0xFF - (pl.col("rtt_us") / 3000).round().clip(0, 0xFF).cast(pl.UInt8)) + df = df.drop("rtt_us") + df = df.with_columns(r=0xFF, g=pl.col("gb"), b=pl.col("gb")) + df = df.drop("gb") + print("done.") + total_size = tiles_per_side * tile_size + print(f"[{z=}] creating image row data...", end=" ", flush=True) + all_rows = np.zeros((total_size, total_size, channels), dtype = "uint8") + all_rows[(df.get_column("y"), df.get_column("x"))] = df.select(["r", "g", "b"]).to_numpy() + all_rows = all_rows.reshape(total_size, total_size * channels) + print("done.") + del df + + z_path = tiles_dir / f"{z}" + z_path.mkdir(exist_ok=True, parents=True) + print(f"[{z=}] creating individual tile data...", end=" ", flush=True) + tile_data = [ + (z_path / f"{y}" / f"{x}.png", all_rows[ + y * tile_size : y * tile_size + tile_size, + x * tile_size_by_channels : x * tile_size_by_channels + tile_size_by_channels + ]) + for x in range(tiles_per_side) + for y in range(tiles_per_side) + ] + print("done.") + print(f"[{z=}] writing {tiles_per_side}x{tiles_per_side}={tiles_per_side * tiles_per_side} images...", end=" ", flush=True) + with Pool(processes) as pool: + pool.starmap(write_tile, tile_data) + print("done.") + del tile_data + del all_rows + + if tiles_per_side == 1: + break + old_tiles_per_side = tiles_per_side + tiles_per_side //= 2 + + print(f"[{z=}] rescaling {len(gdf)} coords from {old_tiles_per_side}x{old_tiles_per_side} to {tiles_per_side}x{tiles_per_side} tiles...", end=" ", flush=True) + new_gdf = gdf.with_columns(x=pl.col("x") // 2, y=pl.col("y") // 2) + del gdf + gdf = new_gdf.group_by(["x", "y"]).mean() + print(f"done. {len(gdf)} coords remaining.") + +def main(): + data_dir = Path("data") + csv_path = data_dir / "full-scan.csv" + parquet_path = data_dir / "full-scan.parquet" + if not parquet_path.exists(): + print(f"no parquet file found at \"{parquet_path}\", generating now...") + convert_to_parquet(csv_path, parquet_path) + tiles_dir = Path("tiles") + generate_tiles(parquet_path, tiles_dir) + +if __name__ == "__main__": + main() diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..361d91c --- /dev/null +++ b/poetry.lock @@ -0,0 +1,125 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "iptools" +version = "0.7.0" +description = "Python utilites for manipulating IPv4 and IPv6 addresses" +optional = false +python-versions = "*" +files = [ + {file = "iptools-0.7.0-py2.py3-none-any.whl", hash = "sha256:a91fc7478fd795ac6b2d47c869fb46db7666ffec817bcb0560ef119e204237f0"}, + {file = "iptools-0.7.0.tar.gz", hash = "sha256:118a4f638bb5fa0123df56fe3be703b112a689167539bcc194f8698ccdd9e2ea"}, +] + +[package.extras] +testing = ["nose (>=1.0)"] + +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "numpy-hilbert-curve" +version = "1.0.1" +description = "Implements Hilbert space-filling curves for Python with numpy" +optional = false +python-versions = "*" +files = [ + {file = "numpy-hilbert-curve-1.0.1.tar.gz", hash = "sha256:0745dbd4c16b258c180342d6df57dfa99110b9d98c86a84d920f29af5cc0707b"}, +] + +[[package]] +name = "polars-lts-cpu" +version = "0.20.17" +description = "Blazingly fast DataFrame library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "polars_lts_cpu-0.20.17-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c5ba1113df88bd0e46bc2e649279f1e2f09f20d24a7e3a8b07d342d1e117bf40"}, + {file = "polars_lts_cpu-0.20.17-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:514e833c63d2734d9028ca754fe441479cb8d68d06efe9f88fdb348db9578941"}, + {file = "polars_lts_cpu-0.20.17-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3512862da0bcb764ed5e63bb122d265295d503e5294c839d5f46f88937543cc1"}, + {file = "polars_lts_cpu-0.20.17-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:2a30789e25a07e0c925e6fde030d2ee53024ae621a0194c423ff83f359d5f62c"}, + {file = "polars_lts_cpu-0.20.17-cp38-abi3-win_amd64.whl", hash = "sha256:b5a3487d481517525d7c9b9c69210f123c2d1f233c47487fa058646c2dc3d42c"}, + {file = "polars_lts_cpu-0.20.17.tar.gz", hash = "sha256:e11eb08f9264459339af4942c4be9c187daf2ffe4040d24284582e4e0e492ab7"}, +] + +[package.extras] +adbc = ["adbc-driver-manager", "adbc-driver-sqlite"] +all = ["polars[adbc,async,cloudpickle,connectorx,deltalake,fastexcel,fsspec,gevent,numpy,pandas,plot,pyarrow,pydantic,pyiceberg,sqlalchemy,timezone,xlsx2csv,xlsxwriter]"] +async = ["nest-asyncio"] +cloudpickle = ["cloudpickle"] +connectorx = ["connectorx (>=0.3.2)"] +deltalake = ["deltalake (>=0.14.0)"] +fastexcel = ["fastexcel (>=0.9)"] +fsspec = ["fsspec"] +gevent = ["gevent"] +matplotlib = ["matplotlib"] +numpy = ["numpy (>=1.16.0)"] +openpyxl = ["openpyxl (>=3.0.0)"] +pandas = ["pandas", "pyarrow (>=7.0.0)"] +plot = ["hvplot (>=0.9.1)"] +pyarrow = ["pyarrow (>=7.0.0)"] +pydantic = ["pydantic"] +pyiceberg = ["pyiceberg (>=0.5.0)"] +pyxlsb = ["pyxlsb (>=1.0)"] +sqlalchemy = ["pandas", "sqlalchemy"] +timezone = ["backports-zoneinfo", "tzdata"] +xlsx2csv = ["xlsx2csv (>=0.8.0)"] +xlsxwriter = ["xlsxwriter"] + +[[package]] +name = "pypng" +version = "0.20220715.0" +description = "Pure Python library for saving and loading PNG images" +optional = false +python-versions = "*" +files = [ + {file = "pypng-0.20220715.0-py3-none-any.whl", hash = "sha256:4a43e969b8f5aaafb2a415536c1a8ec7e341cd6a3f957fd5b5f32a4cfeed902c"}, + {file = "pypng-0.20220715.0.tar.gz", hash = "sha256:739c433ba96f078315de54c0db975aee537cbc3e1d0ae4ed9aab0ca1e427e2c1"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "ae4649713b0931b2549ea9b0951c28123e2c1d7abcdf26bfd202ac2aa039d9db" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..edaed62 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "ipmap" +version = "0.1.0" +description = "" +authors = ["LilyRose2798 "] +license = "AGPLv3" + +[tool.poetry.dependencies] +python = "^3.11" +pypng = "^0.20220715.0" +iptools = "^0.7.0" +numpy = "^1.26.4" +numpy-hilbert-curve = "^1.0.1" +polars-lts-cpu = "^0.20.17" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api"