Compare commits

..

2 Commits

2 changed files with 35 additions and 29 deletions

View File

@ -81,39 +81,15 @@ def generate_tiles(parquet_path: Path, tiles_dir: Path, *, tile_size = default_t
else: else:
raise ValueError(f"unknown variant '{variant}'") raise ValueError(f"unknown variant '{variant}'")
if json_path: if json_path is not None:
if json_path.is_dir(): if json_path.is_dir():
raise ValueError("json path must not be a directory") raise ValueError("json path must not be a directory")
try: try:
tiles_dir_parts = tiles_dir.relative_to(json_path.parent).parts tiles_dir_parts = tiles_dir.relative_to(json_path.parent).parts
except ValueError: except ValueError:
raise ValueError("tiles path must be relative to the json path") raise ValueError("tiles path must be relative to the json path")
try: else:
text = json_path.read_text(encoding = "UTF-8") tiles_dir_parts = None
except:
if not quiet:
print("json file not found at provided path, so it will be created instead")
tile_metadata = {}
else:
try:
tile_metadata: dict = json.loads(text)
except:
raise ValueError("invalid json found at provided path")
tile_metadata_cur = tile_metadata
for part in tiles_dir_parts:
if not part in tile_metadata_cur:
tile_metadata_cur[part] = {}
tile_metadata_cur = tile_metadata_cur[part]
for variant in variants:
if not variant in tile_metadata_cur:
tile_metadata_cur[variant] = colormaps
else:
tile_metadata_cur[variant] = dedup_preserving_order(tile_metadata_cur[variant] + colormaps)
if not quiet:
print(f"writing metadata to json file at '{json_path}'...", end = " ", flush = True)
json_path.write_text(json.dumps(tile_metadata, indent=2), encoding = "UTF-8")
if not quiet:
print("done")
if not quiet: if not quiet:
print(f"reading parquet '{parquet_path}'...", end = " ", flush = True) print(f"reading parquet '{parquet_path}'...", end = " ", flush = True)
@ -187,6 +163,36 @@ def generate_tiles(parquet_path: Path, tiles_dir: Path, *, tile_size = default_t
break break
scale_down_coords() scale_down_coords()
if json_path is not None and tiles_dir_parts is not None:
try:
text = json_path.read_text(encoding = "UTF-8")
except:
if not quiet:
print("json file not found at provided path, so it will be created instead")
tile_metadata = {}
else:
try:
tile_metadata: dict = json.loads(text)
except:
if not quiet:
print("invalid json found at provided path, so re-creating file")
tile_metadata = {}
tile_metadata_cur = tile_metadata
for part in tiles_dir_parts:
if not part in tile_metadata_cur:
tile_metadata_cur[part] = {}
tile_metadata_cur = tile_metadata_cur[part]
for variant in variants:
if not variant in tile_metadata_cur:
tile_metadata_cur[variant] = colormaps
else:
tile_metadata_cur[variant] = dedup_preserving_order(tile_metadata_cur[variant] + colormaps)
if not quiet:
print(f"writing metadata to json file at '{json_path}'...", end = " ", flush = True)
json_path.write_text(json.dumps(tile_metadata, indent=2), encoding = "UTF-8")
if not quiet:
print("done")
def remove_tiles(tiles_dir: Path, *, json_path: Path | None = None, quiet = False): def remove_tiles(tiles_dir: Path, *, json_path: Path | None = None, quiet = False):
if not tiles_dir.is_dir(): if not tiles_dir.is_dir():
raise ValueError(f"'{tiles_dir}' is not an existing directory") raise ValueError(f"'{tiles_dir}' is not an existing directory")

View File

@ -37,6 +37,6 @@ ssh -i "$REMOTE_KEY" "$REMOTE" "
rm '$REMOTE_CSV_PATH' && rm '$REMOTE_CSV_PATH' &&
mkdir -p '$CURRENT_REMOTE_TILES_PATH' && mkdir -p '$CURRENT_REMOTE_TILES_PATH' &&
'$REMOTE_IPMAP_PATH' generate -a -c '$COLORMAPS' -v '$VARIANTS' -j '$REMOTE_JSON_PATH' '$REMOTE_PARQUET_PATH' '$CURRENT_REMOTE_TILES_PATH' '$REMOTE_IPMAP_PATH' generate -a -c '$COLORMAPS' -v '$VARIANTS' -j '$REMOTE_JSON_PATH' '$REMOTE_PARQUET_PATH' '$CURRENT_REMOTE_TILES_PATH'
cd '$REMOTE_DATA_PATH' && for f in *; do (( \$f < $date )) && rm \"\$f\"; done cd '$REMOTE_DATA_PATH' && for f in *; do (( \$f < $KEEP_DATE )) && rm \"\$f\"; done
cd '$REMOTE_TILES_PATH' && for f in *; do (( \$f < $date )) && '$REMOTE_IPMAP_PATH' remove -j '$REMOTE_JSON_PATH' \"\$f\"; done cd '$REMOTE_TILES_PATH' && for f in *; do (( \$f < $KEEP_DATE )) && '$REMOTE_IPMAP_PATH' remove -j '$REMOTE_JSON_PATH' \"\$f\"; done
" "