lily
/
ipmap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: lily:f939fbe2f7f2f7019cfb16a137060db0fda85535
Branches Tags
lily:main
..
compare: lily:28e5196412b49ba1f4efab2ad8ee393103c4654e
Branches Tags
lily:main

No commits in common. "f939fbe2f7f2f7019cfb16a137060db0fda85535" and "28e5196412b49ba1f4efab2ad8ee393103c4654e" have entirely different histories.
f939fbe2f7 ... 28e5196412

2 changed files with 29 additions and 35 deletions
Show Stats Expand all files Collapse all files

60
ipmap.py
View File

@ -81,15 +81,39 @@ def generate_tiles(parquet_path: Path, tiles_dir: Path, *, tile_size = default_t
else: else:
raise ValueError(f"unknown variant '{variant}'") raise ValueError(f"unknown variant '{variant}'")
if json_path is not None: if json_path:
if json_path.is_dir(): if json_path.is_dir():
raise ValueError("json path must not be a directory") raise ValueError("json path must not be a directory")
try: try:
tiles_dir_parts = tiles_dir.relative_to(json_path.parent).parts tiles_dir_parts = tiles_dir.relative_to(json_path.parent).parts
except ValueError: except ValueError:
raise ValueError("tiles path must be relative to the json path") raise ValueError("tiles path must be relative to the json path")
else: try:
tiles_dir_parts = None text = json_path.read_text(encoding = "UTF-8")
except:
if not quiet:
print("json file not found at provided path, so it will be created instead")
tile_metadata = {}
else:
try:
tile_metadata: dict = json.loads(text)
except:
raise ValueError("invalid json found at provided path")
tile_metadata_cur = tile_metadata
for part in tiles_dir_parts:
if not part in tile_metadata_cur:
tile_metadata_cur[part] = {}
tile_metadata_cur = tile_metadata_cur[part]
for variant in variants:
if not variant in tile_metadata_cur:
tile_metadata_cur[variant] = colormaps
else:
tile_metadata_cur[variant] = dedup_preserving_order(tile_metadata_cur[variant] + colormaps)
if not quiet:
print(f"writing metadata to json file at '{json_path}'...", end = " ", flush = True)
json_path.write_text(json.dumps(tile_metadata, indent=2), encoding = "UTF-8")
if not quiet:
print("done")
if not quiet: if not quiet:
print(f"reading parquet '{parquet_path}'...", end = " ", flush = True) print(f"reading parquet '{parquet_path}'...", end = " ", flush = True)
@ -163,36 +187,6 @@ def generate_tiles(parquet_path: Path, tiles_dir: Path, *, tile_size = default_t
break break
scale_down_coords() scale_down_coords()
if json_path is not None and tiles_dir_parts is not None:
try:
text = json_path.read_text(encoding = "UTF-8")
except:
if not quiet:
print("json file not found at provided path, so it will be created instead")
tile_metadata = {}
else:
try:
tile_metadata: dict = json.loads(text)
except:
if not quiet:
print("invalid json found at provided path, so re-creating file")
tile_metadata = {}
tile_metadata_cur = tile_metadata
for part in tiles_dir_parts:
if not part in tile_metadata_cur:
tile_metadata_cur[part] = {}
tile_metadata_cur = tile_metadata_cur[part]
for variant in variants:
if not variant in tile_metadata_cur:
tile_metadata_cur[variant] = colormaps
else:
tile_metadata_cur[variant] = dedup_preserving_order(tile_metadata_cur[variant] + colormaps)
if not quiet:
print(f"writing metadata to json file at '{json_path}'...", end = " ", flush = True)
json_path.write_text(json.dumps(tile_metadata, indent=2), encoding = "UTF-8")
if not quiet:
print("done")
def remove_tiles(tiles_dir: Path, *, json_path: Path | None = None, quiet = False): def remove_tiles(tiles_dir: Path, *, json_path: Path | None = None, quiet = False):
if not tiles_dir.is_dir(): if not tiles_dir.is_dir():
raise ValueError(f"'{tiles_dir}' is not an existing directory") raise ValueError(f"'{tiles_dir}' is not an existing directory")

4
zmap.sh
View File

@ -37,6 +37,6 @@ ssh -i "$REMOTE_KEY" "$REMOTE" "
rm '$REMOTE_CSV_PATH' && rm '$REMOTE_CSV_PATH' &&
mkdir -p '$CURRENT_REMOTE_TILES_PATH' && mkdir -p '$CURRENT_REMOTE_TILES_PATH' &&
'$REMOTE_IPMAP_PATH' generate -a -c '$COLORMAPS' -v '$VARIANTS' -j '$REMOTE_JSON_PATH' '$REMOTE_PARQUET_PATH' '$CURRENT_REMOTE_TILES_PATH' '$REMOTE_IPMAP_PATH' generate -a -c '$COLORMAPS' -v '$VARIANTS' -j '$REMOTE_JSON_PATH' '$REMOTE_PARQUET_PATH' '$CURRENT_REMOTE_TILES_PATH'
cd '$REMOTE_DATA_PATH' && for f in *; do (( \$f < $KEEP_DATE )) && rm \"\$f\"; done cd '$REMOTE_DATA_PATH' && for f in *; do (( \$f < $date )) && rm \"\$f\"; done
cd '$REMOTE_TILES_PATH' && for f in *; do (( \$f < $KEEP_DATE )) && '$REMOTE_IPMAP_PATH' remove -j '$REMOTE_JSON_PATH' \"\$f\"; done cd '$REMOTE_TILES_PATH' && for f in *; do (( \$f < $date )) && '$REMOTE_IPMAP_PATH' remove -j '$REMOTE_JSON_PATH' \"\$f\"; done
" "