diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..11a62cb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +# See https://dev.to/m1yag1/how-to-setup-your-project-with-pre-commit-black-and-flake8-183k +# for pre-commits setup +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pycqa/isort + rev: 6.0.1 + hooks: + - id: isort + args: ["--profile=black"] + name: isort (python) + - repo: https://github.com/ambv/black + rev: 25.1.0 + hooks: + - id: black + language_version: python3.10 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.0.0 + hooks: + - id: flake8 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f4b408..21daa04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # CHANGELOG +- lint +- ajout de pre-commit hooks pour appliquer le lint au moment des commits +- patchwork crée lui-même les sous-dossiers dont chaque étape a besoin +- correctif pour la recherche de correspondance des las dans le csv de matching + ## 1.1.0 - modification de chemin pour pouvoir passer dans la gpao - coupure des chemins de fichiers en chemins de répertoires/nom de fichiers pour pouvoir les utiliser sur docker + store diff --git a/README.md b/README.md index 7f9ab2f..11db08f 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ Patchwork est un outil permettant d'enrichir un fichier lidar à haute densité avec des points d'un fichier à basse densité dans les secteurs où le premier fichier n'a pas de point mais où le second en possède. ## Fonctionnement -Les données en entrée sont: -- un fichier lidar que l'ont souhaite enrichir -- un fichier lidar contenant des points supplémentaires - -En sortie il y a : -- Un fichier, copie du premier en entrée, enrichi des points voulu - +Les données en entrée sont: +- un fichier lidar que l'ont souhaite enrichir +- un fichier lidar contenant des points supplémentaires + +En sortie il y a : +- Un fichier, copie du premier en entrée, enrichi des points voulus + Les deux fichiers d'entrée sont découpés en tuiles carrées, généralement d'1m². Si une tuile du fichier à enrichir ne contient aucun point ayant le classement qui nous intéresse, on prend les points de la tuile de même emplacement du fichier de points supplémentaire. L'appartenance à une tuile est décidée par un arrondi par défaut, c'est-à-dire que tous les éléments de [n, n+1[ (ouvert en n+1) font parti de la même tuile. @@ -30,31 +30,31 @@ Le script d'ajout de points peut être lancé via : ``` python main.py filepath.DONOR_FILE=[chemin fichier donneur] filepath.RECIPIENT_FILE=[chemin fichier receveur] filepath.OUTPUT_FILE=[chemin fichier de sortie] [autres options] ``` -Les différentes options, modifiables soit dans le fichierconfigs/configs_patchwork.yaml, soit en ligne de commande comme indiqué juste au-dessus : - -filepath.DONOR_DIRECTORY : Le répertoire du fichier qui peut donner des points à ajouter -filepath.DONOR_NAME : Le nom du fichier qui peut donner des points à ajouter -filepath.RECIPIENT_DIRECTORY : Le répertoire du fichier qui va obtenir des points en plus -filepath.RECIPIENT_NAME : Le nom du fichier qui va obtenir des points en plus -filepath.OUTPUT_DIR : Le répertoire du fichier en sortie -filepath.OUTPUT_NAME : Le nom du fichier en sortie -filepath.OUTPUT_INDICES_MAP_DIR : Le répertoire de sortie du fichier d'indice -filepath.OUTPUT_INDICES_MAP_NAME : Le nom de sortie du fichier d'indice - -DONOR_CLASS_LIST : Défaut [2, 9]. La liste des classes des points du fichier donneur qui peuvent être ajoutés. +Les différentes options, modifiables soit dans le fichier `configs/configs_patchwork.yaml`, soit en ligne de commande comme indiqué juste au-dessus : + +filepath.DONOR_DIRECTORY : Le répertoire du fichier qui peut donner des points à ajouter +filepath.DONOR_NAME : Le nom du fichier qui peut donner des points à ajouter +filepath.RECIPIENT_DIRECTORY : Le répertoire du fichier qui va obtenir des points en plus +filepath.RECIPIENT_NAME : Le nom du fichier qui va obtenir des points en plus +filepath.OUTPUT_DIR : Le répertoire du fichier en sortie +filepath.OUTPUT_NAME : Le nom du fichier en sortie +filepath.OUTPUT_INDICES_MAP_DIR : Le répertoire de sortie du fichier d'indice +filepath.OUTPUT_INDICES_MAP_NAME : Le nom de sortie du fichier d'indice + +DONOR_CLASS_LIST : Défaut [2, 22]. La liste des classes des points du fichier donneur qui peuvent être ajoutés. RECIPIENT_CLASS_LIST : Défaut [2, 3, 9, 17]. La liste des classes des points du fichier receveur qui, s'ils sont absents dans une cellule, justifirons de prendre les points du fichier donneur de la même cellule TILE_SIZE : Défaut 1000. Taille du côté de l'emprise carrée représentée par les fichiers lidar d'entrée -PATCH_SIZE : Défaut 1. taille en mètre du côté d'une cellule (doit être un diviseur de TILE_SIZE, soit pour 1000 : 0.25, 0.5, 2, 4, 5, 10, 25...) +PATCH_SIZE : Défaut 1. taille en mètre du côté d'une cellule (doit être un diviseur de TILE_SIZE, soit pour 1000 : 0.25, 0.5, 2, 4, 5, 10, 25...) Le script de sélection/découpe de fichier lidar peut être lancé via : ``` -python lidar_filepath.py filepath.DONOR_DIRECTORY=[répertoire_fichiers_donneurs] filepath.RECIPIENT_DIRECTORY=[répertoire_fichiers_receveurs] filepath.SHP_NAME=[nom_shapefile] filepath.SHP_DIRECTORY=[répertoire_shapefile] filepath.CSV_NAME=[nom_fichier_csv] filepath.CSV_DIRECTORY=[répertoire_fichier_csv] filepath.OUTPUT_DIRECTORY=[chemin_de_sortie] +python lidar_selecter.py filepath.DONOR_DIRECTORY=[répertoire_fichiers_donneurs] filepath.RECIPIENT_DIRECTORY=[répertoire_fichiers_receveurs] filepath.SHP_NAME=[nom_shapefile] filepath.SHP_DIRECTORY=[répertoire_shapefile] filepath.CSV_NAME=[nom_fichier_csv] filepath.CSV_DIRECTORY=[répertoire_fichier_csv] filepath.OUTPUT_DIRECTORY=[chemin_de_sortie] ``` -filepath.DONOR_DIRECTORY: Le répertoire contenant les fichiers lidar donneurs -filepath.RECIPIENT_DIRECTORY: Le répertoire contenant les fichiers lidar receveurs -filepath.SHP_NAME: Le nom du shapefile contenant l'emprise du chantier qui délimite les fichiers lidar qui nous intéressent -filepath.SHP_DIRECTORY: Le répertoire du fichier shapefile -filepath.CSV_NAME: Le nom du fichier csv qui lie les différents fichiers donneurs et receveurs -filepath.CSV_DIRECTORY: Le répertoire du fichier csv -filepath.OUTPUT_DIRECTORY: le répertoire recevant les fichiers lidar découpés \ No newline at end of file +filepath.DONOR_DIRECTORY: Le répertoire contenant les fichiers lidar donneurs +filepath.RECIPIENT_DIRECTORY: Le répertoire contenant les fichiers lidar receveurs +filepath.SHP_NAME: Le nom du shapefile contenant l'emprise du chantier qui délimite les fichiers lidar qui nous intéressent +filepath.SHP_DIRECTORY: Le répertoire du fichier shapefile +filepath.CSV_NAME: Le nom du fichier csv qui lie les différents fichiers donneurs et receveurs +filepath.CSV_DIRECTORY: Le répertoire du fichier csv +filepath.OUTPUT_DIRECTORY: le répertoire recevant les fichiers lidar découpés \ No newline at end of file diff --git a/constants.py b/constants.py deleted file mode 100644 index 6a15205..0000000 --- a/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -# INTERNAL CONSTANTS, NOT TO BE CHANGED -CLASSIFICATION_STR = 'classification' -PATCH_X_STR = 'patch_x' -PATCH_Y_STR = 'patch_y' -DONOR_SUBDIRECTORY_NAME = "donor" -RECIPIENT_SUBDIRECTORY_NAME = "recipient" - -COORDINATES_KEY = 'coordinates' -DONOR_FILE_KEY = 'donor_file' -RECIPIENT_FILE_KEY = 'recipient_file' diff --git a/environment.yml b/environment.yml index 22116ed..b1afd58 100644 --- a/environment.yml +++ b/environment.yml @@ -9,11 +9,13 @@ dependencies: - numpy - geopandas==0.* - shapely>=2.0.3 + - rasterio # ------------- logging ------------- # - loguru # --------- hydra configs --------- # - hydra-core - hydra-colorlog # ----------- linting --------------- # - - flake8 - - rasterio \ No newline at end of file + - pre-commit + - black + - flake8 \ No newline at end of file diff --git a/main.py b/main.py index b7910df..d19fc2c 100644 --- a/main.py +++ b/main.py @@ -3,7 +3,7 @@ import hydra from omegaconf import DictConfig -from patchwork import patchwork +from patchwork.patchwork import patchwork @hydra.main(config_path="configs/", config_name="configs_patchwork.yaml", version_base="1.2") diff --git a/patchwork/constants.py b/patchwork/constants.py new file mode 100644 index 0000000..260b86b --- /dev/null +++ b/patchwork/constants.py @@ -0,0 +1,10 @@ +# INTERNAL CONSTANTS, NOT TO BE CHANGED +CLASSIFICATION_STR = "classification" +PATCH_X_STR = "patch_x" +PATCH_Y_STR = "patch_y" +DONOR_SUBDIRECTORY_NAME = "donor" +RECIPIENT_SUBDIRECTORY_NAME = "recipient" + +COORDINATES_KEY = "coordinates" +DONOR_FILE_KEY = "donor_file" +RECIPIENT_FILE_KEY = "recipient_file" diff --git a/indices_map.py b/patchwork/indices_map.py similarity index 75% rename from indices_map.py rename to patchwork/indices_map.py index 214b83d..fb3f473 100644 --- a/indices_map.py +++ b/patchwork/indices_map.py @@ -1,18 +1,18 @@ -import os +import os import numpy as np -from omegaconf import DictConfig -import rasterio as rs -from rasterio.transform import from_origin import pandas as pd +import rasterio as rs +from omegaconf import DictConfig from pandas import DataFrame +from rasterio.transform import from_origin -from tools import get_tile_origin_from_pointcloud -from constants import PATCH_X_STR, PATCH_Y_STR +from patchwork.constants import PATCH_X_STR, PATCH_Y_STR +from patchwork.tools import get_tile_origin_from_pointcloud def create_indices_grid(config: DictConfig, df_points: DataFrame) -> np.ndarray: - """ create a binary grid matching the tile the points of df_points are from, where each patch is equal to: + """create a binary grid matching the tile the points of df_points are from, where each patch is equal to: 1 if the patch has at least one point of df_points 0 if the patch has no point from df_points """ @@ -40,14 +40,23 @@ def create_indices_map(config: DictConfig, df_points: DataFrame): corner_x, corner_y = get_tile_origin_from_pointcloud(config, df_points) grid = create_indices_grid(config, df_points) - output_indices_map_path = os.path.join(config.filepath.OUTPUT_INDICES_MAP_DIR, config.filepath.OUTPUT_INDICES_MAP_NAME) + os.makedirs(config.filepath.OUTPUT_INDICES_MAP_DIR, exist_ok=True) + output_indices_map_path = os.path.join( + config.filepath.OUTPUT_INDICES_MAP_DIR, config.filepath.OUTPUT_INDICES_MAP_NAME + ) transform = from_origin(corner_x, corner_y, config.PATCH_SIZE, config.PATCH_SIZE) - indices_map = rs.open(output_indices_map_path, 'w', driver='GTiff', - height=grid.shape[0], width=grid.shape[1], - count=1, dtype=str(grid.dtype), - crs=config.CRS, - transform=transform) + indices_map = rs.open( + output_indices_map_path, + "w", + driver="GTiff", + height=grid.shape[0], + width=grid.shape[1], + count=1, + dtype=str(grid.dtype), + crs=config.CRS, + transform=transform, + ) indices_map.write(grid, 1) indices_map.close() diff --git a/lidar_selecter.py b/patchwork/lidar_selecter.py similarity index 72% rename from lidar_selecter.py rename to patchwork/lidar_selecter.py index 925b634..192bd88 100644 --- a/lidar_selecter.py +++ b/patchwork/lidar_selecter.py @@ -2,59 +2,62 @@ import pathlib import timeit -import hydra -from omegaconf import DictConfig import geopandas as gpd +import hydra import laspy -from laspy import ScaleAwarePointRecord -from shapely import box import numpy as np +from laspy import ScaleAwarePointRecord +from loguru import logger +from omegaconf import DictConfig from pandas import DataFrame +from shapely import box from shapely.geometry import MultiPolygon from shapely.vectorized import contains -from loguru import logger -import constants as c -from tools import identify_bounds, get_tile_origin_from_pointcloud, crop_tile +import patchwork.constants as c +from patchwork.tools import crop_tile, get_tile_origin_from_pointcloud, identify_bounds -@hydra.main(config_path="configs/", config_name="configs_patchwork.yaml", version_base="1.2") +@hydra.main(config_path="../configs/", config_name="configs_patchwork.yaml", version_base="1.2") def patchwork_dispatcher(config: DictConfig): - data = {c.COORDINATES_KEY: [], - c.DONOR_FILE_KEY: [], - c.RECIPIENT_FILE_KEY: [] - } + data = {c.COORDINATES_KEY: [], c.DONOR_FILE_KEY: [], c.RECIPIENT_FILE_KEY: []} df_result = DataFrame(data=data) # preparing donor files: - select_lidar(config, - config.filepath.DONOR_DIRECTORY, - config.filepath.OUTPUT_DIRECTORY, - c.DONOR_SUBDIRECTORY_NAME, - df_result, - c.DONOR_FILE_KEY, - True - ) + select_lidar( + config, + config.filepath.DONOR_DIRECTORY, + config.filepath.OUTPUT_DIRECTORY, + c.DONOR_SUBDIRECTORY_NAME, + df_result, + c.DONOR_FILE_KEY, + True, + ) # preparing recipient files: - select_lidar(config, - config.filepath.RECIPIENT_DIRECTORY, - config.filepath.OUTPUT_DIRECTORY, - c.RECIPIENT_SUBDIRECTORY_NAME, - df_result, - c.RECIPIENT_FILE_KEY, - False, - ) - df_result.to_csv(os.path.join(config.filepath.CSV_DIRECTORY, config.filepath.CSV_NAME), index=False) + select_lidar( + config, + config.filepath.RECIPIENT_DIRECTORY, + config.filepath.OUTPUT_DIRECTORY, + c.RECIPIENT_SUBDIRECTORY_NAME, + df_result, + c.RECIPIENT_FILE_KEY, + False, + ) + + pathlib.Path(config.filepath.CSV_DIRECTORY).mkdir(exist_ok=True) + df_result.to_csv( + os.path.join(config.filepath.CSV_DIRECTORY, config.filepath.CSV_NAME), index=False, encoding="utf-8" + ) def cut_lidar(las_points: ScaleAwarePointRecord, shapefile_geometry: MultiPolygon) -> ScaleAwarePointRecord: - shapefile_contains_mask = contains(shapefile_geometry, np.array(las_points['x']), np.array(las_points['y'])) + shapefile_contains_mask = contains(shapefile_geometry, np.array(las_points["x"]), np.array(las_points["y"])) return las_points[shapefile_contains_mask] def update_df_result(df_result: DataFrame, df_key: str, corner_string: str, file_path: str): # corner_string not yet in df_result - if not corner_string in list(df_result[c.COORDINATES_KEY]): - new_row = {c.COORDINATES_KEY:corner_string, c.DONOR_FILE_KEY: "", c.RECIPIENT_FILE_KEY:""} + if corner_string not in list(df_result[c.COORDINATES_KEY]): + new_row = {c.COORDINATES_KEY: corner_string, c.DONOR_FILE_KEY: "", c.RECIPIENT_FILE_KEY: ""} new_row[df_key] = file_path df_result.loc[len(df_result)] = new_row return df_result @@ -64,13 +67,15 @@ def update_df_result(df_result: DataFrame, df_key: str, corner_string: str, file return df_result -def select_lidar(config: DictConfig, - input_directory:str, - output_directory:str, - subdirectory_name: str, - df_result:DataFrame, - df_key: str, - to_be_cut: bool): +def select_lidar( + config: DictConfig, + input_directory: str, + output_directory: str, + subdirectory_name: str, + df_result: DataFrame, + df_key: str, + to_be_cut: bool, +): """ Walk the input directory searching for las files, and pick the ones that intersect with the shapefile. When a las file is half inside the shapfile, it is cut if "to_be_cut" is true, otherwise it kept whole @@ -83,6 +88,13 @@ def select_lidar(config: DictConfig, time_old = timeit.default_timer() time_start = time_old + + directory_path = os.path.join(output_directory, subdirectory_name) + # Create output dir only if asked to cut + # Otherwise the input file is intended to be used directly (no copy to the output directory) + if to_be_cut: + pathlib.Path(directory_path).mkdir(parents=True, exist_ok=True) + for root, _, file_names in os.walk(input_directory): for file_name in file_names: @@ -95,7 +107,6 @@ def select_lidar(config: DictConfig, raw_las_points = las_file.read().points min_x, max_x, min_y, max_y = identify_bounds(config.TILE_SIZE, raw_las_points) intersect_area = shapefile_geometry.intersection(box(min_x, min_y, max_x, max_y)).area - # if intersect area == 0, this tile is fully outside the shapefile if intersect_area == 0: @@ -105,9 +116,6 @@ def select_lidar(config: DictConfig, time_old = time_new continue - directory_path = os.path.join(output_directory, subdirectory_name) - pathlib.Path(directory_path).mkdir(parents=True, exist_ok=True) - las_points = crop_tile(config, raw_las_points) x_corner, y_corner = get_tile_origin_from_pointcloud(config, las_points) diff --git a/patchwork.py b/patchwork/patchwork.py similarity index 64% rename from patchwork.py rename to patchwork/patchwork.py index b9eea01..2fef4bf 100644 --- a/patchwork.py +++ b/patchwork/patchwork.py @@ -1,40 +1,38 @@ - -from shutil import copy2 -from typing import List, Tuple import os from pathlib import Path +from shutil import copy2 +from typing import List, Tuple -from omegaconf import DictConfig - +import laspy import numpy as np import pandas as pd -import laspy -from laspy import ScaleAwarePointRecord, LasReader +from laspy import LasReader, ScaleAwarePointRecord +from omegaconf import DictConfig -import constants as c -from tools import get_tile_origin_from_pointcloud, crop_tile -from indices_map import create_indices_map -from constants import CLASSIFICATION_STR, PATCH_X_STR, PATCH_Y_STR +import patchwork.constants as c +from patchwork.indices_map import create_indices_map +from patchwork.tools import crop_tile, get_tile_origin_from_pointcloud -def get_selected_classes_points(config: DictConfig, - tile_origin: Tuple[int, int], - points_list: ScaleAwarePointRecord, - class_list: list[int], - fields_to_keep: list[str]) -> pd.DataFrame: - """get a list of points from a las, and return a ndarray of those point with the selected classification - """ +def get_selected_classes_points( + config: DictConfig, + tile_origin: Tuple[int, int], + points_list: ScaleAwarePointRecord, + class_list: list[int], + fields_to_keep: list[str], +) -> pd.DataFrame: + """get a list of points from a las, and return a ndarray of those point with the selected classification""" # we add automatically classification, so we remove it if it's in field_to_keep - if CLASSIFICATION_STR in fields_to_keep: - fields_to_keep.remove(CLASSIFICATION_STR) + if c.CLASSIFICATION_STR in fields_to_keep: + fields_to_keep.remove(c.CLASSIFICATION_STR) table_fields_to_keep = [points_list[field] for field in fields_to_keep] table_field_necessary = [ np.int32(points_list.x / config.PATCH_SIZE), # convert x into the coordinate of the patch np.int32(points_list.y / config.PATCH_SIZE), # convert y into the coordinate of the patch - points_list.classification - ] + points_list.classification, + ] all_classes_points = np.array(table_fields_to_keep + table_field_necessary).transpose() @@ -42,20 +40,20 @@ def get_selected_classes_points(config: DictConfig, for classification in class_list: mask = mask | (all_classes_points[:, -1] == classification) wanted_classes_points = all_classes_points[mask] - all_fields_list = [*fields_to_keep, PATCH_X_STR, PATCH_Y_STR, CLASSIFICATION_STR] + all_fields_list = [*fields_to_keep, c.PATCH_X_STR, c.PATCH_Y_STR, c.CLASSIFICATION_STR] df_wanted_classes_points = pd.DataFrame(wanted_classes_points, columns=all_fields_list) # "push" the points on the limit of the tile to the closest patch - mask_points_on_max_x = df_wanted_classes_points[PATCH_X_STR] == tile_origin[0] + config.TILE_SIZE - df_wanted_classes_points.loc[mask_points_on_max_x, PATCH_X_STR] = tile_origin[0] + config.TILE_SIZE - 1 - mask_points_on_max_y = df_wanted_classes_points[PATCH_Y_STR] == tile_origin[1] - df_wanted_classes_points.loc[mask_points_on_max_y, PATCH_Y_STR] = tile_origin[1] - 1 + mask_points_on_max_x = df_wanted_classes_points[c.PATCH_X_STR] == tile_origin[0] + config.TILE_SIZE + df_wanted_classes_points.loc[mask_points_on_max_x, c.PATCH_X_STR] = tile_origin[0] + config.TILE_SIZE - 1 + mask_points_on_max_y = df_wanted_classes_points[c.PATCH_Y_STR] == tile_origin[1] + df_wanted_classes_points.loc[mask_points_on_max_y, c.PATCH_Y_STR] = tile_origin[1] - 1 return df_wanted_classes_points def get_type(new_column_size: int): - """ return the type matching the new_column_size (must be in [8,16,32,64])""" + """return the type matching the new_column_size (must be in [8,16,32,64])""" match new_column_size: case 8: return np.int8 @@ -74,8 +72,7 @@ def get_complementary_points(config: DictConfig) -> pd.DataFrame: donor_file_path = os.path.join(donor_dir, donor_name) recipient_file_path = os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME) - with laspy.open(donor_file_path) as donor_file, \ - laspy.open(recipient_file_path) as recipient_file: + with laspy.open(donor_file_path) as donor_file, laspy.open(recipient_file_path) as recipient_file: raw_donor_points = donor_file.read().points donor_points = crop_tile(config, raw_donor_points) raw_recipient_points = recipient_file.read().points @@ -85,41 +82,40 @@ def get_complementary_points(config: DictConfig) -> pd.DataFrame: tile_origin_donor = get_tile_origin_from_pointcloud(config, donor_points) tile_origin_recipient = get_tile_origin_from_pointcloud(config, recipient_points) if tile_origin_donor != tile_origin_recipient: - raise ValueError(f"{donor_file_path} and \ - {recipient_file_path} are not on the same area") + raise ValueError( + f"{donor_file_path} and \ + {recipient_file_path} are not on the same area" + ) donor_columns = get_field_from_header(donor_file) - df_donor_points = get_selected_classes_points(config, - tile_origin_donor, - donor_points, - config.DONOR_CLASS_LIST, - donor_columns - ) - df_recipient_points = get_selected_classes_points(config, - tile_origin_recipient, - recipient_points, - config.RECIPIENT_CLASS_LIST, - [] - ) + df_donor_points = get_selected_classes_points( + config, tile_origin_donor, donor_points, config.DONOR_CLASS_LIST, donor_columns + ) + df_recipient_points = get_selected_classes_points( + config, tile_origin_recipient, recipient_points, config.RECIPIENT_CLASS_LIST, [] + ) # set, for each patch of coordinate (patch_x, patch_y), the number of recipient point # should have no record for when count == 0, therefore "df_recipient_non_empty_patches" list all # and only the patches with at least a point # In other words, the next column should be filled with "False" everywhere - df_recipient_non_empty_patches = df_recipient_points.groupby(by=[PATCH_X_STR, PATCH_Y_STR]).count().classification == 0 + df_recipient_non_empty_patches = ( + df_recipient_points.groupby(by=[c.PATCH_X_STR, c.PATCH_Y_STR]).count().classification == 0 + ) # for each (patch_x,patch_y) patch, we join to a donor point the count of recipient points on that patch # since it's a left join, it keeps all the left record (all the donor points) # and put a "NaN" if the recipient point count is null (no record) - joined_patches = pd.merge(df_donor_points, - df_recipient_non_empty_patches, - on=[PATCH_X_STR, PATCH_Y_STR], - how='left', - suffixes=('', config.RECIPIENT_SUFFIX) - ) + joined_patches = pd.merge( + df_donor_points, + df_recipient_non_empty_patches, + on=[c.PATCH_X_STR, c.PATCH_Y_STR], + how="left", + suffixes=("", config.RECIPIENT_SUFFIX), + ) # only keep donor points in patches where there is no recipient point - return joined_patches.loc[joined_patches[CLASSIFICATION_STR + config.RECIPIENT_SUFFIX].isnull()] + return joined_patches.loc[joined_patches[c.CLASSIFICATION_STR + config.RECIPIENT_SUFFIX].isnull()] def get_field_from_header(las_file: LasReader) -> List[str]: @@ -137,22 +133,24 @@ def test_field_exists(file_path: str, colmun: str) -> bool: def append_points(config: DictConfig, extra_points: pd.DataFrame): # get field to copy : recipient_filepath = os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME) - ouput_filepath = os.path.join(config.filepath.OUTPUT_DIR, config.filepath.OUTPUT_NAME) + ouput_filepath = os.path.join(config.filepath.OUTPUT_DIR, config.filepath.OUTPUT_NAME) with laspy.open(recipient_filepath) as recipient_file: recipient_fields_list = get_field_from_header(recipient_file) # get fields that are in the donor file we can transmit to the recipient without problem # classification is in the fields to exclude because it will be copy in a special way - fields_to_exclude = [PATCH_X_STR, - PATCH_Y_STR, - CLASSIFICATION_STR, - CLASSIFICATION_STR + config.RECIPIENT_SUFFIX - ] - - fields_to_keep = [field for field in recipient_fields_list if - (field.lower() in extra_points.columns) - and (field.lower() not in fields_to_exclude) - ] + fields_to_exclude = [ + c.PATCH_X_STR, + c.PATCH_Y_STR, + c.CLASSIFICATION_STR, + c.CLASSIFICATION_STR + config.RECIPIENT_SUFFIX, + ] + + fields_to_keep = [ + field + for field in recipient_fields_list + if (field.lower() in extra_points.columns) and (field.lower() not in fields_to_exclude) + ] copy2(recipient_filepath, ouput_filepath) @@ -162,8 +160,10 @@ def append_points(config: DictConfig, extra_points: pd.DataFrame): # if we want a new column, we start by adding its name if config.NEW_COLUMN: if test_field_exists(recipient_filepath, config.NEW_COLUMN): - raise ValueError(f"{config.NEW_COLUMN} already exists as \ - column name in {recipient_filepath}") + raise ValueError( + f"{config.NEW_COLUMN} already exists as \ + column name in {recipient_filepath}" + ) new_column_type = get_type(config.NEW_COLUMN_SIZE) output_las = laspy.read(ouput_filepath) output_las.add_extra_dim(laspy.ExtraBytesParams(name=config.NEW_COLUMN, type=new_column_type)) @@ -179,18 +179,19 @@ def append_points(config: DictConfig, extra_points: pd.DataFrame): # translate the classification values: for classification in config.DONOR_CLASS_LIST: new_classification = config.VIRTUAL_CLASS_TRANSLATION[classification] - extra_points.loc[extra_points[CLASSIFICATION_STR] == classification, CLASSIFICATION_STR] \ - = new_classification + extra_points.loc[extra_points[c.CLASSIFICATION_STR] == classification, c.CLASSIFICATION_STR] = ( + new_classification + ) else: extra_points[config.NEW_COLUMN] = config.VALUE_ADDED_POINTS new_points[config.NEW_COLUMN] = extra_points[config.NEW_COLUMN] - new_points.classification = extra_points[CLASSIFICATION_STR] + new_points.classification = extra_points[c.CLASSIFICATION_STR] output_las.append_points(new_points) -def get_donor_from_csv(recipient_file_path:str, csv_file_path:str)-> str: +def get_donor_from_csv(recipient_file_path: str, csv_file_path: str) -> str: """ check if there is a donor file, in the csv file, matching the recipient file return the path to that file if it exists @@ -198,21 +199,27 @@ def get_donor_from_csv(recipient_file_path:str, csv_file_path:str)-> str: """ df_csv_data = pd.read_csv(csv_file_path) donor_file_paths = df_csv_data.loc[df_csv_data[c.RECIPIENT_FILE_KEY] == recipient_file_path, c.DONOR_FILE_KEY] - if len(donor_file_paths) > 0: - return donor_file_paths.loc[0] # there should be only one donor file for a given recipient file - return "" + if len(donor_file_paths) == 1: + return donor_file_paths.iloc[0] + elif len(donor_file_paths) == 0: + return "" + else: + raise RuntimeError( + f"Found more than one donor file associated with recipient file {recipient_file_path}." + "Please check the matching csv file" + ) def get_donor_path(config: DictConfig) -> Tuple[str, str]: """Return a donor directory and a name: If there is no csv file provided in config, return DONOR_DIRECTORY and DONOR_NAME if there is a csv file provided, return DONOR_DIRECTORY and DONOR_NAME matching the given RECIPIENT - if there is a csv file provided but no matching DONOR, return "" twice """ - if config.filepath.CSV_DIRECTORY and config.filepath.CSV_NAME : + if there is a csv file provided but no matching DONOR, return "" twice""" + if config.filepath.CSV_DIRECTORY and config.filepath.CSV_NAME: csv_file_path = os.path.join(config.filepath.CSV_DIRECTORY, config.filepath.CSV_NAME) recipient_file_path = os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME) donor_file_path = get_donor_from_csv(recipient_file_path, csv_file_path) - if not donor_file_path: # if there is no matching donor file, we do nothing + if not donor_file_path: # if there is no matching donor file, we do nothing return "", "" return str(Path(donor_file_path).parent), str(Path(donor_file_path).name) return config.filepath.DONOR_DIRECTORY, config.filepath.DONOR_NAME @@ -220,9 +227,9 @@ def get_donor_path(config: DictConfig) -> Tuple[str, str]: def patchwork(config: DictConfig): _, donor_name = get_donor_path(config) - if not donor_name: # if no matching donor, we simply copy the recipient to the output without doing anything + if not donor_name: # if no matching donor, we simply copy the recipient to the output without doing anything recipient_filepath = os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME) - ouput_filepath = os.path.join(config.filepath.OUTPUT_DIR, config.filepath.OUTPUT_NAME) + ouput_filepath = os.path.join(config.filepath.OUTPUT_DIR, config.filepath.OUTPUT_NAME) copy2(recipient_filepath, ouput_filepath) return diff --git a/tools.py b/patchwork/tools.py similarity index 81% rename from tools.py rename to patchwork/tools.py index a711e0b..313e646 100644 --- a/tools.py +++ b/patchwork/tools.py @@ -1,8 +1,7 @@ -import numpy as np from typing import Tuple +import numpy as np from laspy import ScaleAwarePointRecord - from omegaconf import DictConfig @@ -11,10 +10,10 @@ def get_tile_origin_from_pointcloud(config: DictConfig, points: np.ndarray | Sca if not len(points): raise ValueError("No points to determine the coordinate of the tile") - x_min = np.min(points['x'], axis=0) - x_max = np.max(points['x'], axis=0) - y_min = np.min(points['y'], axis=0) - y_max = np.max(points['y'], axis=0) + x_min = np.min(points["x"], axis=0) + x_max = np.max(points["x"], axis=0) + y_min = np.min(points["y"], axis=0) + y_max = np.max(points["y"], axis=0) length_tile_x = x_max / config.TILE_SIZE - np.floor(x_min / config.TILE_SIZE) length_tile_y = np.ceil(y_max / config.TILE_SIZE) - y_min / config.TILE_SIZE @@ -32,8 +31,8 @@ def get_tile_origin_from_pointcloud(config: DictConfig, points: np.ndarray | Sca def identify_bounds(tile_size: float, points: ScaleAwarePointRecord) -> Tuple[int, int, int, int]: """Return the bounds of a tile represented by its points""" - gravity_x = np.sum(points['x']) / len(points) - gravity_y = np.sum(points['y']) / len(points) + gravity_x = np.sum(points["x"]) / len(points) + gravity_y = np.sum(points["y"]) / len(points) min_x = int(gravity_x / tile_size) * tile_size max_x = int(gravity_x / tile_size) * tile_size + tile_size min_y = int(gravity_y / tile_size) * tile_size @@ -45,4 +44,4 @@ def identify_bounds(tile_size: float, points: ScaleAwarePointRecord) -> Tuple[in def crop_tile(config: DictConfig, points: ScaleAwarePointRecord) -> np.ndarray: """Crop points to the tile containing the center of gravity""" min_x, max_x, min_y, max_y = identify_bounds(config.TILE_SIZE, points) - return points[(points['x'] >= min_x) & (points['x'] <= max_x) & (points['y'] >= min_y) & (points['y'] <= max_y)] + return points[(points["x"] >= min_x) & (points["x"] <= max_x) & (points["y"] >= min_y) & (points["y"] <= max_y)] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..76dae3a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "patchwork" + +description = "Merge patches from 2 lidar files depending on the points classes" +readme = "README.md" +authors = [ + { name = "Michel Daab" }, + { name = "Léa Vauchier", email = "lea.vauchier@ign.fr" }, +] + +[tool.black] +line-length = 119 +include = '\.pyi?$' +exclude = ''' +/( + \.toml + |\.sh + |\.git + |\.ini + |\.bat + | data +)/ +''' + +[tool.pytest.ini_options] +markers = ["slow: marks tests as slow (select with '--runslow')"] diff --git a/test/data/donor_more_fields_test.las b/test/data/donor_more_fields_test.las new file mode 100644 index 0000000..0746628 Binary files /dev/null and b/test/data/donor_more_fields_test.las differ diff --git a/test/data/recipient_more_fields_test.laz b/test/data/recipient_more_fields_test.laz new file mode 100644 index 0000000..39ba6fc Binary files /dev/null and b/test/data/recipient_more_fields_test.laz differ diff --git a/test/data/recipient_slided_test.laz b/test/data/recipient_slided_test.laz new file mode 100644 index 0000000..6a8dd3c Binary files /dev/null and b/test/data/recipient_slided_test.laz differ diff --git a/test/test_indices_map.py b/test/test_indices_map.py index 27bbc88..6e6a664 100644 --- a/test/test_indices_map.py +++ b/test/test_indices_map.py @@ -1,21 +1,22 @@ -import sys import os -from hydra import compose, initialize import numpy as np import pandas as pd import rasterio as rs +from hydra import compose, initialize from rasterio.transform import from_origin -sys.path.append('../patchwork') - -from indices_map import create_indices_grid, create_indices_map, read_indices_map -from constants import PATCH_X_STR, PATCH_Y_STR +from patchwork.constants import PATCH_X_STR, PATCH_Y_STR +from patchwork.indices_map import ( + create_indices_grid, + create_indices_map, + read_indices_map, +) PATCH_SIZE = 1 TILE_SIZE = 3 -DATA_POINTS = {'x': [0.0, 1.5, 3, 1.5, 2.5], 'y': [0.0, 0.5, 0.5, 1.5, 3]} +DATA_POINTS = {"x": [0.0, 1.5, 3, 1.5, 2.5], "y": [0.0, 0.5, 0.5, 1.5, 3]} # we want y=0 at the bottom, but in a ndarray it's at the top, so grid['y'] = SIZE_Y - data_points['y'] POINTS_IN_GRID = [(0, 2), (1, 2), (2, 2), (1, 1), (2, 0)] POINTS_NOT_IN_GRID = [(0, 1), (2, 1), (0, 0), (0, 1)] @@ -24,11 +25,7 @@ def test_create_indices_points(): with initialize(version_base="1.2", config_path="../configs"): config = compose( - config_name="configs_patchwork.yaml", - overrides=[ - f"PATCH_SIZE={PATCH_SIZE}", - f"TILE_SIZE={TILE_SIZE}" - ] + config_name="configs_patchwork.yaml", overrides=[f"PATCH_SIZE={PATCH_SIZE}", f"TILE_SIZE={TILE_SIZE}"] ) df_points = pd.DataFrame(data=DATA_POINTS) grid = create_indices_grid(config, df_points) @@ -53,7 +50,7 @@ def test_create_indices_map(tmp_path_factory): f"TILE_SIZE={TILE_SIZE}", f"filepath.OUTPUT_INDICES_MAP_DIR={tmp_file_dir}", f"filepath.OUTPUT_INDICES_MAP_NAME={tmp_file_name}", - ] + ], ) df_points = pd.DataFrame(data=DATA_POINTS) @@ -81,26 +78,32 @@ def test_read_indices_map(tmp_path_factory): f"TILE_SIZE={TILE_SIZE}", f"filepath.INPUT_INDICES_MAP_DIR={tmp_file_dir}", f"filepath.INPUT_INDICES_MAP_NAME={tmp_file_name}", - ] + ], ) - grid = np.array([ - [0, 0, 1], - [0, 1, 0], - [1, 1, 1],]) + grid = np.array( + [ + [0, 0, 1], + [0, 1, 0], + [1, 1, 1], + ] + ) transform = from_origin(0, 3, config.PATCH_SIZE, config.PATCH_SIZE) - output_indices_map_path = os.path.join(config.filepath.INPUT_INDICES_MAP_DIR, config.filepath.INPUT_INDICES_MAP_NAME) - indices_map = rs.open(output_indices_map_path, - 'w', - driver='GTiff', - height=grid.shape[0], - width=grid.shape[1], - count=1, - dtype=str(grid.dtype), - crs=config.CRS, - transform=transform - ) + output_indices_map_path = os.path.join( + config.filepath.INPUT_INDICES_MAP_DIR, config.filepath.INPUT_INDICES_MAP_NAME + ) + indices_map = rs.open( + output_indices_map_path, + "w", + driver="GTiff", + height=grid.shape[0], + width=grid.shape[1], + count=1, + dtype=str(grid.dtype), + crs=config.CRS, + transform=transform, + ) indices_map.write(grid, 1) indices_map.close() diff --git a/test/test_lidar_selecter.py b/test/test_lidar_selecter.py index 7538e88..dbcb0e8 100644 --- a/test/test_lidar_selecter.py +++ b/test/test_lidar_selecter.py @@ -1,18 +1,12 @@ -import sys - +import geopandas as gpd import laspy -from hydra import compose, initialize -from shapely.geometry import MultiPolygon import numpy as np -import geopandas as gpd +from hydra import compose, initialize from pandas import DataFrame +from shapely.geometry import MultiPolygon -import constants as c - -sys.path.append('../patchwork') - -from lidar_selecter import cut_lidar, select_lidar, update_df_result - +import patchwork.constants as c +from patchwork.lidar_selecter import cut_lidar, select_lidar, update_df_result CRS = 2154 TILE_SIZE = 1000 @@ -32,10 +26,15 @@ def test_cut_lidar(): - shapefile_geometry = MultiPolygon([([SHAPE_CORNER_1, SHAPE_CORNER_2, SHAPE_CORNER_3],),]) - las_points = np.array([POINT_INSIDE_1, POINT_INSIDE_2, POINT_OUTSIDE_1, POINT_OUTSIDE_2], - dtype=[('x', 'float32'), ('y', 'float32'), ('z', 'float32')] - ) + shapefile_geometry = MultiPolygon( + [ + ([SHAPE_CORNER_1, SHAPE_CORNER_2, SHAPE_CORNER_3],), + ] + ) + las_points = np.array( + [POINT_INSIDE_1, POINT_INSIDE_2, POINT_OUTSIDE_1, POINT_OUTSIDE_2], + dtype=[("x", "float32"), ("y", "float32"), ("z", "float32")], + ) points_in_geometry = cut_lidar(las_points, shapefile_geometry) list_points_in_geometry = points_in_geometry.tolist() assert POINT_INSIDE_1 in list_points_in_geometry @@ -50,8 +49,12 @@ def test_select_lidar(tmp_path_factory): shp_name = "shapefile.shp" shapefile_path = shp_dir / shp_name - shapefile_geometry = MultiPolygon([([SHAPE_CORNER_1, SHAPE_CORNER_2, SHAPE_CORNER_3],),]) - gpd_shapefile_geometry = gpd.GeoDataFrame({'geometry': [shapefile_geometry]}, crs=CRS) + shapefile_geometry = MultiPolygon( + [ + ([SHAPE_CORNER_1, SHAPE_CORNER_2, SHAPE_CORNER_3],), + ] + ) + gpd_shapefile_geometry = gpd.GeoDataFrame({"geometry": [shapefile_geometry]}, crs=CRS) gpd_shapefile_geometry.to_file(shapefile_path) # las creation @@ -59,13 +62,14 @@ def test_select_lidar(tmp_path_factory): las_path = input_directory / LASFILE_NAME las = laspy.LasData(laspy.LasHeader(point_format=3, version="1.4")) - las_points = np.array([POINT_INSIDE_1, POINT_INSIDE_2, POINT_OUTSIDE_1, POINT_OUTSIDE_2], - dtype=[('x', 'float32'), ('y', 'float32'), ('z', 'float32')] - ) + las_points = np.array( + [POINT_INSIDE_1, POINT_INSIDE_2, POINT_OUTSIDE_1, POINT_OUTSIDE_2], + dtype=[("x", "float32"), ("y", "float32"), ("z", "float32")], + ) - las.x = las_points['x'] - las.y = las_points['y'] - las.z = las_points['z'] + las.x = las_points["x"] + las.y = las_points["y"] + las.z = las_points["z"] las.write(las_path) @@ -73,23 +77,16 @@ def test_select_lidar(tmp_path_factory): output_directory = las_path = tmp_path_factory.mktemp(OUTPUT_DIRECTORY) # create teh dataframe (to put the result in) - data = {c.COORDINATES_KEY: [], - c.DONOR_FILE_KEY: [], - c.RECIPIENT_FILE_KEY: [] - } + data = {c.COORDINATES_KEY: [], c.DONOR_FILE_KEY: [], c.RECIPIENT_FILE_KEY: []} df_result = DataFrame(data=data) with initialize(version_base="1.2", config_path="../configs"): config = compose( config_name="configs_patchwork.yaml", - overrides=[ - f"filepath.SHP_DIRECTORY={shp_dir}", - f"filepath.SHP_NAME={shp_name}", - f"TILE_SIZE={TILE_SIZE}" - ] + overrides=[f"filepath.SHP_DIRECTORY={shp_dir}", f"filepath.SHP_NAME={shp_name}", f"TILE_SIZE={TILE_SIZE}"], ) subdirectory_name = SUBDIRECTORY_NAME - select_lidar(config, input_directory, output_directory, subdirectory_name,df_result, c.DONOR_FILE_KEY, True) + select_lidar(config, input_directory, output_directory, subdirectory_name, df_result, c.DONOR_FILE_KEY, True) output_las_path = output_directory / subdirectory_name / LASFILE_NAME with laspy.open(output_las_path) as las_file: @@ -107,10 +104,7 @@ def test_select_lidar(tmp_path_factory): def test_update_df_result(): - data = {c.COORDINATES_KEY: [], - c.DONOR_FILE_KEY: [], - c.RECIPIENT_FILE_KEY: [] - } + data = {c.COORDINATES_KEY: [], c.DONOR_FILE_KEY: [], c.RECIPIENT_FILE_KEY: []} df_result = DataFrame(data=data) corner_string = "1111_2222" donor_path = "dummy_path_1" diff --git a/test/test_patchwork.py b/test/test_patchwork.py index 760239c..2f042d7 100644 --- a/test/test_patchwork.py +++ b/test/test_patchwork.py @@ -1,20 +1,23 @@ -import sys import os -import pytest -from hydra import compose, initialize import laspy import numpy as np import pandas as pd +import pytest +from hydra import compose, initialize from pandas import DataFrame -sys.path.append('../patchwork') - -import constants as c -from patchwork import get_complementary_points, get_field_from_header, get_selected_classes_points -from patchwork import get_type, append_points, get_donor_from_csv, get_donor_path -from tools import get_tile_origin_from_pointcloud -from constants import CLASSIFICATION_STR +import patchwork.constants as c +from patchwork.patchwork import ( + append_points, + get_complementary_points, + get_donor_from_csv, + get_donor_path, + get_field_from_header, + get_selected_classes_points, + get_type, +) +from patchwork.tools import get_tile_origin_from_pointcloud RECIPIENT_TEST_DIR = "test/data/" RECIPIENT_TEST_NAME = "recipient_test.laz" @@ -22,8 +25,8 @@ DONOR_CLASS_LIST = [2, 9] RECIPIENT_CLASS_LIST = [2, 3, 9, 17] VIRTUAL_CLASS_TRANSLATION = {2: 69, 9: 70} -POINT_1 = {'x': 1, 'y': 2, 'z': 3, CLASSIFICATION_STR: 4} -POINT_2 = {'x': 5, 'y': 6, 'z': 7, CLASSIFICATION_STR: 8} +POINT_1 = {"x": 1, "y": 2, "z": 3, c.CLASSIFICATION_STR: 4} +POINT_2 = {"x": 5, "y": 6, "z": 7, c.CLASSIFICATION_STR: 8} NEW_COLUMN = "virtual_column" NEW_COLUMN_SIZE = 8 VALUE_ADDED_POINTS = 1 @@ -44,7 +47,6 @@ COORDINATES = "1234_6789" - def test_get_field_from_header(): with laspy.open(os.path.join(RECIPIENT_TEST_DIR, RECIPIENT_TEST_NAME)) as recipient_file: recipient_fields_list = get_field_from_header(recipient_file) @@ -61,23 +63,21 @@ def test_get_selected_classes_points(): overrides=[ f"filepath.RECIPIENT_DIRECTORY={RECIPIENT_TEST_DIR}", f"filepath.RECIPIENT_NAME={RECIPIENT_TEST_NAME}", - f"RECIPIENT_CLASS_LIST={RECIPIENT_CLASS_LIST}" - ] + f"RECIPIENT_CLASS_LIST={RECIPIENT_CLASS_LIST}", + ], ) - with laspy.open(os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME)) as recipient_file: + with laspy.open( + os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME) + ) as recipient_file: recipient_points = recipient_file.read().points tile_origin_recipient = get_tile_origin_from_pointcloud(config, recipient_points) df_recipient_points = get_selected_classes_points( - config, - tile_origin_recipient, - recipient_points, - config.RECIPIENT_CLASS_LIST, - [] - ) - for classification in np.unique(df_recipient_points[CLASSIFICATION_STR]): + config, tile_origin_recipient, recipient_points, config.RECIPIENT_CLASS_LIST, [] + ) + for classification in np.unique(df_recipient_points[c.CLASSIFICATION_STR]): assert classification in RECIPIENT_CLASS_LIST @@ -93,7 +93,7 @@ def test_get_complementary_points(): f"DONOR_CLASS_LIST={DONOR_CLASS_LIST}", f"RECIPIENT_CLASS_LIST={RECIPIENT_CLASS_LIST}", f"+VIRTUAL_CLASS_TRANSLATION={VIRTUAL_CLASS_TRANSLATION}", - ] + ], ) complementary_points = get_complementary_points(config) @@ -125,7 +125,7 @@ def test_get_complementary_points_2(): f"DONOR_CLASS_LIST={DONOR_CLASS_LIST}", f"RECIPIENT_CLASS_LIST={RECIPIENT_CLASS_LIST}", f"+VIRTUAL_CLASS_TRANSLATION={VIRTUAL_CLASS_TRANSLATION}", - ] + ], ) complementary_points = get_complementary_points(config) @@ -148,11 +148,11 @@ def test_get_complementary_points_3(): f"filepath.DONOR_NAME={DONOR_TEST_NAME}", f"filepath.RECIPIENT_DIRECTORY={RECIPIENT_SLIDED_TEST_DIR}", f"filepath.RECIPIENT_NAME={RECIPIENT_SLIDED_TEST_NAME}", - ] + ], ) las = laspy.read(os.path.join(RECIPIENT_TEST_DIR, RECIPIENT_TEST_NAME)) - las.points['x'] = las.points['x'] + config.TILE_SIZE + las.points["x"] = las.points["x"] + config.TILE_SIZE las.write(os.path.join(RECIPIENT_SLIDED_TEST_DIR, RECIPIENT_SLIDED_TEST_NAME)) with pytest.raises(Exception): @@ -185,7 +185,7 @@ def test_append_points(tmp_path_factory): f"filepath.RECIPIENT_NAME={RECIPIENT_TEST_NAME}", f"filepath.OUTPUT_DIR={tmp_file_dir}", f"filepath.OUTPUT_NAME={tmp_file_name}", - ] + ], ) recipient_file_path = os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME) @@ -211,7 +211,11 @@ def test_append_points(tmp_path_factory): assert point in las_output.points # add 1 point - extra_points = pd.DataFrame(data=[POINT_1, ]) + extra_points = pd.DataFrame( + data=[ + POINT_1, + ] + ) append_points(config, extra_points) # assert a point has been added @@ -219,7 +223,7 @@ def test_append_points(tmp_path_factory): assert get_point_count(output_file) == point_count + 1 # # add 0 point - extra_points = pd.DataFrame(data={'x': [], 'y': [], 'z': [], CLASSIFICATION_STR: []}) + extra_points = pd.DataFrame(data={"x": [], "y": [], "z": [], c.CLASSIFICATION_STR: []}) append_points(config, extra_points) # assert a point has been added @@ -241,8 +245,8 @@ def test_append_points_new_column(tmp_path_factory): f"filepath.OUTPUT_NAME={tmp_file_name}", f"NEW_COLUMN={NEW_COLUMN}", f"NEW_COLUMN_SIZE={NEW_COLUMN_SIZE}", - f"VALUE_ADDED_POINTS={VALUE_ADDED_POINTS}" - ] + f"VALUE_ADDED_POINTS={VALUE_ADDED_POINTS}", + ], ) output_file = os.path.join(config.filepath.OUTPUT_DIR, config.filepath.OUTPUT_NAME) @@ -251,7 +255,9 @@ def test_append_points_new_column(tmp_path_factory): append_points(config, extra_points) # assert a point has been added - point_count = get_point_count(os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME)) + point_count = get_point_count( + os.path.join(config.filepath.RECIPIENT_DIRECTORY, config.filepath.RECIPIENT_NAME) + ) assert get_point_count(output_file) == point_count + 2 # assert the new column is here @@ -265,19 +271,28 @@ def test_append_points_new_column(tmp_path_factory): assert new_column[-2] == VALUE_ADDED_POINTS assert max(new_column[:-2]) == 0 + def test_get_donor_from_csv(tmp_path_factory): csv_file_path = tmp_path_factory.mktemp("csv") / "recipients_donors_links.csv" donor_more_fields_test_path = os.path.join(DONOR_MORE_FIELDS_TEST_DIR, DONOR_MORE_FIELDS_TEST_NAME) recipient_more_fields_test_path = os.path.join(RECIPIENT_TEST_DIR, RECIPIENT_TEST_NAME) - data = {c.COORDINATES_KEY: [COORDINATES, ], - c.DONOR_FILE_KEY: [donor_more_fields_test_path, ], - c.RECIPIENT_FILE_KEY: [recipient_more_fields_test_path, ] - } + data = { + c.COORDINATES_KEY: [ + COORDINATES, + ], + c.DONOR_FILE_KEY: [ + donor_more_fields_test_path, + ], + c.RECIPIENT_FILE_KEY: [ + recipient_more_fields_test_path, + ], + } DataFrame(data=data).to_csv(csv_file_path) donor_file_path = get_donor_from_csv(recipient_more_fields_test_path, csv_file_path) assert donor_file_path == donor_more_fields_test_path + def test_get_donor_path(tmp_path_factory): # check get_donor_path when no csv with initialize(version_base="1.2", config_path="../configs"): @@ -288,7 +303,7 @@ def test_get_donor_path(tmp_path_factory): f"filepath.DONOR_NAME={DONOR_TEST_NAME}", f"filepath.RECIPIENT_DIRECTORY={RECIPIENT_SLIDED_TEST_DIR}", f"filepath.RECIPIENT_NAME={RECIPIENT_SLIDED_TEST_NAME}", - ] + ], ) donor_dir, donor_name = get_donor_path(config) assert donor_dir == DONOR_TEST_DIR @@ -299,10 +314,7 @@ def test_get_donor_path(tmp_path_factory): csv_file_name = "recipients_donors_links.csv" csv_file_path = os.path.join(csv_file_dir, csv_file_name) - data = {c.COORDINATES_KEY: [], - c.DONOR_FILE_KEY: [], - c.RECIPIENT_FILE_KEY: [] - } + data = {c.COORDINATES_KEY: [], c.DONOR_FILE_KEY: [], c.RECIPIENT_FILE_KEY: []} DataFrame(data=data).to_csv(csv_file_path) with initialize(version_base="1.2", config_path="../configs"): @@ -315,7 +327,7 @@ def test_get_donor_path(tmp_path_factory): f"filepath.RECIPIENT_NAME={RECIPIENT_SLIDED_TEST_NAME}", f"filepath.CSV_DIRECTORY={csv_file_dir}", f"filepath.CSV_NAME={csv_file_name}", - ] + ], ) donor_dir, donor_name = get_donor_path(config) @@ -325,10 +337,17 @@ def test_get_donor_path(tmp_path_factory): # check get_donor_path when csv but with a matching donor in it donor_more_fields_test_path = os.path.join(DONOR_MORE_FIELDS_TEST_DIR, DONOR_MORE_FIELDS_TEST_NAME) recipient_more_fields_test_path = os.path.join(RECIPIENT_TEST_DIR, RECIPIENT_TEST_NAME) - data = {c.COORDINATES_KEY: [COORDINATES, ], - c.DONOR_FILE_KEY: [donor_more_fields_test_path, ], - c.RECIPIENT_FILE_KEY: [recipient_more_fields_test_path, ] - } + data = { + c.COORDINATES_KEY: [ + COORDINATES, + ], + c.DONOR_FILE_KEY: [ + donor_more_fields_test_path, + ], + c.RECIPIENT_FILE_KEY: [ + recipient_more_fields_test_path, + ], + } DataFrame(data=data).to_csv(csv_file_path) with initialize(version_base="1.2", config_path="../configs"): @@ -341,7 +360,7 @@ def test_get_donor_path(tmp_path_factory): f"filepath.RECIPIENT_NAME={RECIPIENT_TEST_NAME}", f"filepath.CSV_DIRECTORY={csv_file_dir}", f"filepath.CSV_NAME={csv_file_name}", - ] + ], ) donor_dir, donor_name = get_donor_path(config) diff --git a/test/test_tools.py b/test/test_tools.py index 19765ff..58514f6 100644 --- a/test/test_tools.py +++ b/test/test_tools.py @@ -1,12 +1,8 @@ -import sys - import numpy as np -from hydra import compose, initialize import pytest +from hydra import compose, initialize -sys.path.append('../patchwork') - -from tools import get_tile_origin_from_pointcloud +from patchwork.tools import get_tile_origin_from_pointcloud TILE_SIZE = 1000 @@ -18,13 +14,13 @@ def test_get_tile_origin_from_pointcloud(): config_name="configs_patchwork.yaml", overrides=[ f"TILE_SIZE={TILE_SIZE}", - ] + ], ) # basic test list_x = [1100, 1500] list_y = [2200, 2800] - points = np.core.records.fromarrays([list_x, list_y], names='x,y') + points = np.core.records.fromarrays([list_x, list_y], names="x,y") corner_x, corner_y = get_tile_origin_from_pointcloud(config, points) assert corner_x == 1000 @@ -33,7 +29,7 @@ def test_get_tile_origin_from_pointcloud(): # limit test 1 list_x = [1000, 2000] list_y = [1000, 2000] - points = np.core.records.fromarrays([list_x, list_y], names='x,y') + points = np.core.records.fromarrays([list_x, list_y], names="x,y") corner_x, corner_y = get_tile_origin_from_pointcloud(config, points) assert corner_x == 1000 @@ -42,7 +38,7 @@ def test_get_tile_origin_from_pointcloud(): # limit test 2 list_x = [1500] list_y = [2300] - points = np.core.records.fromarrays([list_x, list_y], names='x,y') + points = np.core.records.fromarrays([list_x, list_y], names="x,y") corner_x, corner_y = get_tile_origin_from_pointcloud(config, points) assert corner_x == 1000 @@ -51,7 +47,7 @@ def test_get_tile_origin_from_pointcloud(): # limit test 3 list_x = [] list_y = [] - points = np.core.records.fromarrays([list_x, list_y], names='x,y') + points = np.core.records.fromarrays([list_x, list_y], names="x,y") with pytest.raises(ValueError): get_tile_origin_from_pointcloud(config, points) @@ -59,7 +55,7 @@ def test_get_tile_origin_from_pointcloud(): # failed test list_x = [1100, 1500] list_y = [2200, 3800] - points = np.core.records.fromarrays([list_x, list_y], names='x,y') + points = np.core.records.fromarrays([list_x, list_y], names="x,y") with pytest.raises(ValueError): get_tile_origin_from_pointcloud(config, points)