From 60ed38e74a1e4ca1233c6cb065d62a739e0c9e97 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 8 Dec 2020 08:39:15 -0600 Subject: [PATCH 001/161] Testing fixups (#78) * lint * build-option * don't use pip * syntax * need cython * package_data test * include_package_data * no package_data * exclude static libs * CMAKE_INSTALL_DIR * remove cmake_process * PROJECT_NAME * package_data * try CMAKE_INSTALL_LIBDIR * target install * CMAKE_INSTALL_LIBDIR * LIBDIR * target install * CMAKE_INSTALL_DIR * extension location * PROJECT_NAME * CMAKE_INSTALL_DIR * targets * target churn * target churnin * another swing * lint * churnin * chern * NO_POLICY_SCOPE * de-macro * lint * install to PACKAGE_NAME * DESTINATION churn * target churn * target churns * targets for scikit build * reorganize to match scikit-build expected layout * gtest --- .github/workflows/build.yml | 9 +- CMakeLists.txt | 315 ++++++++++++++++++++++++++++++++++++ pdal/CMakeLists.txt | 146 +---------------- pdal/macros.cmake | 2 +- setup.py | 3 +- 5 files changed, 327 insertions(+), 148 deletions(-) create mode 100644 CMakeLists.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b8c7fe4..59ec8882 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,20 +40,25 @@ jobs: - name: Dependencies shell: bash -l {0} run: | - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} compilers ninja pdal pytest conda-build -y + conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} compilers cython ninja pdal pytest conda-build -y - name: Install shell: bash -l {0} run: | - pip install . --install-option="-DWITH_TESTS=ON" + python setup.py install -- -G "Ninja" -DWITH_TESTS=ON + pdal --drivers + +# pip install . --build-option="-DWITH_TESTS=ON" + - name: Test shell: bash -l {0} run: | pip install -e . SKPATH=$(python -c "import skbuild; print (skbuild.constants.SKBUILD_DIR())")/cmake-build export PDAL_DRIVER_PATH=$SKPATH + ls $SKPATH $SKPATH/pdal_filters_python_test $SKPATH/pdal_io_numpy_test py.test diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..07e72a97 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,315 @@ +cmake_minimum_required(VERSION 3.11.0) +project(PDAL) + +# macros for creating targets +#include(${PROJECT_SOURCE_DIR}/macros.cmake NO_POLICY_SCOPE) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +option(WITH_TESTS "Enable tests" OFF) +set(CMAKE_BUILD_TYPE "Release") + + +# Python-finding settings +set(Python3_FIND_STRATEGY "LOCATION") +set(Python3_FIND_REGISTRY "LAST") +set(Python3_FIND_FRAMEWORK "LAST") +find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) + +# find PDAL. Require 2.1+ +find_package(PDAL 2.1 REQUIRED) + +if(SKBUILD) + find_package(PythonExtensions REQUIRED) + find_package(Cython REQUIRED) + message(STATUS "The project is built using scikit-build") +endif() + +execute_process( + COMMAND + ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('Py_ENABLE_SHARED'))" + OUTPUT_VARIABLE Py_ENABLE_SHARED + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +#execute_process( +# COMMAND +# ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('LDSHARED').split(' ', 1)[1])" +# OUTPUT_VARIABLE PYTHON_LDSHARED +# OUTPUT_STRIP_TRAILING_WHITESPACE +#) + +#if (NOT Py_ENABLE_SHARED) +# message(STATUS "Python ${Python3_EXECUTABLE} is statically linked") +# if (APPLE) +# # conda gives us -bundle, which isn't valid +# message(STATUS "Removing extra -bundle argument from sysconfig.get_config_var('LDSHARED')") +# string(REPLACE "-bundle" "" PYTHON_LDSHARED "${PYTHON_LDSHARED}") +# string(STRIP ${PYTHON_LDSHARED} PYTHON_LDSHARED) +# endif() +# # set(Python3_LIBRARIES ${PYTHON_LDSHARED}) +# message(STATUS "Setting Python3_LIBRARIES to '${Python3_LIBRARIES}' due to static Python") +#endif() + +add_subdirectory(pdal) +#set(EXTENSION_SRC +# ./pdal/PyArray.cpp +# ./pdal/PyArray.hpp +# ./pdal/PyDimension.hpp +# ./pdal/PyPipeline.cpp +# ./pdal/PyPipeline.hpp) +# +#set(extension "libpdalpython") +#add_cython_target(${extension} "./pdal/libpdalpython.pyx" CXX PY3) +# +#add_library(${extension} MODULE ${EXTENSION_SRC} libpdalpython) +#target_include_directories( ${extension} +# PRIVATE +# . +# ${PDAL_INCLUDE_DIRS} +# ${Python3_INCLUDE_DIRS} +# ${Python3_NumPy_INCLUDE_DIRS}) +# +#target_link_libraries(${extension} ${PDAL_LIBRARIES}) +#python_extension_module(${extension}) +# +#install(TARGETS ${extension} LIBRARY DESTINATION "${PROJECT_NAME}/pdal") + + +# Taken and adapted from PDAL's cmake macros.cmake + +function(pdal_python_target_compile_settings target) + set_property(TARGET ${target} PROPERTY CXX_STANDARD 11) + set_property(TARGET ${target} PROPERTY CXX_STANDARD_REQUIRED TRUE) + target_compile_definitions(${target} PRIVATE + -DWIN32_LEAN_AND_MEAN) + if (MSVC) + # check for MSVC 8+ + if (NOT (MSVC_VERSION VERSION_LESS 1400)) + target_compile_definitions(${target} PRIVATE + -D_CRT_SECURE_NO_DEPRECATE + -D_CRT_SECURE_NO_WARNINGS + -D_CRT_NONSTDC_NO_WARNING + -D_SCL_SECURE_NO_WARNINGS + ) + target_compile_options(${target} PRIVATE + # Yes, we don't understand GCC pragmas + /wd4068 + # Nitro makes use of Exception Specifications, which results in + # numerous warnings when compiling in MSVC. We will ignore + # them for now. + /wd4290 + /wd4800 + # Windows warns about integer narrowing like crazy and it's + # annoying. In most cases the programmer knows what they're + # doing. A good static analysis tool would be better than + # turning this warning off. + /wd4267 + # Annoying warning about function hiding with virtual + # inheritance. + /wd4250 + # some templates don't return +# /wd4716 + # unwind semantics +# /wd4530 + # Standard C++-type exception handling. + /EHsc + ) + endif() + + endif() +endfunction() + + +############################################################################### +# Add a plugin target. +# _name The plugin name. +# ARGN : +# FILES the source files for the plugin +# LINK_WITH link plugin with libraries +# INCLUDES header directories +# +# The "generate_dimension_hpp" ensures that Dimension.hpp is built before +# attempting to build anything else in the "library". +# +# NOTE: _name is the name of a variable that will hold the plugin name +# when the macro completes +macro(PDAL_PYTHON_ADD_PLUGIN _name _type _shortname) + set(options) + set(oneValueArgs) + set(multiValueArgs FILES LINK_WITH INCLUDES SYSTEM_INCLUDES COMPILE_OPTIONS) + cmake_parse_arguments(PDAL_PYTHON_ADD_PLUGIN "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN}) + if(WIN32) + set(WINSOCK_LIBRARY ws2_32) + set(${_name} "libpdal_plugin_${_type}_${_shortname}") + else() + set(${_name} "pdal_plugin_${_type}_${_shortname}") + endif() + + + add_library(${${_name}} SHARED ${PDAL_PYTHON_ADD_PLUGIN_FILES}) + pdal_python_target_compile_settings(${${_name}}) + target_include_directories(${${_name}} PRIVATE + ${PROJECT_BINARY_DIR}/include + ${PDAL_INCLUDE_DIR} + ${PDAL_PYTHON_ADD_PLUGIN_INCLUDES} + ) + target_link_options(${${_name}} BEFORE PRIVATE ${PDAL_PYTHON_ADD_PLUGIN_COMPILE_OPTIONS}) + target_compile_definitions(${${_name}} PRIVATE + PDAL_PYTHON_LIBRARY="${PYTHON_LIBRARY}" PDAL_DLL_EXPORT) + target_compile_definitions(${${_name}} PRIVATE PDAL_DLL_EXPORT) + if (PDAL_PYTHON_ADD_PLUGIN_SYSTEM_INCLUDES) + target_include_directories(${${_name}} SYSTEM PRIVATE + ${PDAL_PYTHON_ADD_PLUGIN_SYSTEM_INCLUDES}) + endif() + target_link_libraries(${${_name}} + PRIVATE + ${PDAL_PYTHON_ADD_PLUGIN_LINK_WITH} + ${WINSOCK_LIBRARY} + ) + + message(STATUS "PROJECT_NAME: ${PROJECT_NAME}") + install(TARGETS ${${_name}} + LIBRARY DESTINATION "pdal" + ) + if (APPLE) + set_target_properties(${${_name}} PROPERTIES + INSTALL_NAME_DIR "@rpath") + endif() +endmacro(PDAL_PYTHON_ADD_PLUGIN) + + +macro(PDAL_PYTHON_ADD_TEST _name) + set(options) + set(oneValueArgs) + set(multiValueArgs FILES LINK_WITH INCLUDES SYSTEM_INCLUDES) + cmake_parse_arguments(PDAL_PYTHON_ADD_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if (WIN32) + set(WINSOCK_LIBRARY ws2_32) + endif() + add_executable(${_name} ${PDAL_PYTHON_ADD_TEST_FILES}) + + pdal_python_target_compile_settings(${_name}) + target_include_directories(${_name} PRIVATE + ${PDAL_PYTHON_ADD_TEST_INCLUDES}) + if (PDAL_PYTHON_ADD_TEST_SYSTEM_INCLUDES) + target_include_directories(${_name} SYSTEM PRIVATE + ${PDAL_PYTHON_ADD_TEST_SYSTEM_INCLUDES}) + endif() + set_property(TARGET ${_name} PROPERTY FOLDER "Tests") + target_link_libraries(${_name} + PRIVATE + ${PDAL_PYTHON_ADD_TEST_LINK_WITH} + gtest + ${WINSOCK_LIBRARY} + ) + target_compile_definitions(${_name} PRIVATE + PDAL_PYTHON_LIBRARY="${PYTHON_LIBRARY}") + add_test(NAME ${_name} + COMMAND + "${PROJECT_BINARY_DIR}/bin/${_name}" + WORKING_DIRECTORY + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/..") + # Ensure plugins are loaded from build dir + # https://github.com/PDAL/PDAL/issues/840 +# if (WIN32) +# set_property(TEST ${_name} PROPERTY ENVIRONMENT +# "PDAL_DRIVER_PATH=${PROJECT_BINARY_DIR}/bin") +# else() +# set_property(TEST ${_name} PROPERTY ENVIRONMENT +# "PDAL_DRIVER_PATH=${PROJECT_BINARY_DIR}/lib") +# endif() +endmacro(PDAL_PYTHON_ADD_TEST) + + +PDAL_PYTHON_ADD_PLUGIN(numpy_reader reader numpy + FILES + ./pdal/io/NumpyReader.cpp + ./pdal/io/NumpyReader.hpp + ./pdal/plang/Invocation.cpp + ./pdal/plang/Environment.cpp + ./pdal/plang/Redirector.cpp + ./pdal/plang/Script.cpp + LINK_WITH + ${PDAL_LIBRARIES} + ${Python3_LIBRARIES} + ${CMAKE_DL_LIBS} + SYSTEM_INCLUDES + ${PDAL_INCLUDE_DIRS} + ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} + COMPILE_OPTIONS + ${PYTHON_LINK_LIBRARY} + ) + +PDAL_PYTHON_ADD_PLUGIN(python_filter filter python + FILES + ./pdal/filters/PythonFilter.cpp + ./pdal/filters/PythonFilter.hpp + ./pdal/plang/Invocation.cpp + ./pdal/plang/Environment.cpp + ./pdal/plang/Redirector.cpp + ./pdal/plang/Script.cpp + LINK_WITH + ${PDAL_LIBRARIES} + ${Python3_LIBRARIES} + ${CMAKE_DL_LIBS} + SYSTEM_INCLUDES + ${PDAL_INCLUDE_DIRS} + ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} + COMPILE_OPTIONS + ${PYTHON_LINK_LIBRARY} + ) + + +if (WITH_TESTS) + enable_testing() + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + add_subdirectory(pdal/test/gtest) + enable_testing() + include_directories(pdal/test/gtest/include .. ${CMAKE_CURRENT_BINARY_DIR}) + + + + + PDAL_PYTHON_ADD_TEST(pdal_io_numpy_test + FILES + ./pdal/test/NumpyReaderTest.cpp + ./pdal/test/Support.cpp + ./pdal/plang/Invocation.cpp + ./pdal/plang/Environment.cpp + ./pdal/plang/Redirector.cpp + ./pdal/plang/Script.cpp + LINK_WITH + ${numpy_reader} + ${Python3_LIBRARIES} + ${PDAL_LIBRARIES} + ${CMAKE_DL_LIBS} + SYSTEM_INCLUDES + ${PDAL_INCLUDE_DIRS} + ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} + ) + + PDAL_PYTHON_ADD_TEST(pdal_filters_python_test + FILES + ./pdal/test/PythonFilterTest.cpp + ./pdal/test/Support.cpp + ./pdal/plang/Invocation.cpp + ./pdal/plang/Environment.cpp + ./pdal/plang/Redirector.cpp + ./pdal/plang/Script.cpp + LINK_WITH + ${python_filter} + ${Python3_LIBRARIES} + ${PDAL_LIBRARIES} + ${CMAKE_DL_LIBS} + SYSTEM_INCLUDES + ${PDAL_INCLUDE_DIRS} + ${Python3_INCLUDE_DIRS} + ${Python3_NumPy_INCLUDE_DIRS} + ) +endif (WITH_TESTS) diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt index 8eeb3122..97bc051a 100644 --- a/pdal/CMakeLists.txt +++ b/pdal/CMakeLists.txt @@ -1,57 +1,3 @@ -cmake_minimum_required(VERSION 3.11.0) -project(pdal) - -# macros for creating targets -include(${PROJECT_SOURCE_DIR}/macros.cmake NO_POLICY_SCOPE) - -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) -option(WITH_TESTS "Enable tests" OFF) -set(CMAKE_BUILD_TYPE "Release") - - -# Python-finding settings -set(Python3_FIND_STRATEGY "LOCATION") -set(Python3_FIND_REGISTRY "LAST") -set(Python3_FIND_FRAMEWORK "LAST") -find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) - -# find PDAL. Require 2.1+ -find_package(PDAL 2.1 REQUIRED) - -if(SKBUILD) - find_package(PythonExtensions REQUIRED) - find_package(Cython REQUIRED) - message(STATUS "The project is built using scikit-build") -endif() - -execute_process( - COMMAND - ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('Py_ENABLE_SHARED'))" - OUTPUT_VARIABLE Py_ENABLE_SHARED - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -#execute_process( -# COMMAND -# ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('LDSHARED').split(' ', 1)[1])" -# OUTPUT_VARIABLE PYTHON_LDSHARED -# OUTPUT_STRIP_TRAILING_WHITESPACE -#) - -#if (NOT Py_ENABLE_SHARED) -# message(STATUS "Python ${Python3_EXECUTABLE} is statically linked") -# if (APPLE) -# # conda gives us -bundle, which isn't valid -# message(STATUS "Removing extra -bundle argument from sysconfig.get_config_var('LDSHARED')") -# string(REPLACE "-bundle" "" PYTHON_LDSHARED "${PYTHON_LDSHARED}") -# string(STRIP ${PYTHON_LDSHARED} PYTHON_LDSHARED) -# endif() -# # set(Python3_LIBRARIES ${PYTHON_LDSHARED}) -# message(STATUS "Setting Python3_LIBRARIES to '${Python3_LIBRARIES}' due to static Python") -#endif() - set(EXTENSION_SRC PyArray.cpp PyArray.hpp @@ -60,7 +6,7 @@ set(EXTENSION_SRC PyPipeline.hpp) set(extension "libpdalpython") -add_cython_target(${extension} libpdalpython.pyx CXX PY3) +add_cython_target(${extension} "libpdalpython.pyx" CXX PY3) add_library(${extension} MODULE ${EXTENSION_SRC} libpdalpython) target_include_directories( ${extension} @@ -73,94 +19,6 @@ target_include_directories( ${extension} target_link_libraries(${extension} ${PDAL_LIBRARIES}) python_extension_module(${extension}) -install(TARGETS ${extension} LIBRARY DESTINATION ${PROJECT_NAME}) - -PDAL_PYTHON_ADD_PLUGIN(numpy_reader reader numpy - FILES - ./io/NumpyReader.cpp - ./io/NumpyReader.hpp - ./plang/Invocation.cpp - ./plang/Environment.cpp - ./plang/Redirector.cpp - ./plang/Script.cpp - LINK_WITH - ${PDAL_LIBRARIES} - ${Python3_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - COMPILE_OPTIONS - ${PYTHON_LINK_LIBRARY} - ) - -PDAL_PYTHON_ADD_PLUGIN(python_filter filter python - FILES - ./filters/PythonFilter.cpp - ./filters/PythonFilter.hpp - ./plang/Invocation.cpp - ./plang/Environment.cpp - ./plang/Redirector.cpp - ./plang/Script.cpp - LINK_WITH - ${PDAL_LIBRARIES} - ${Python3_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - COMPILE_OPTIONS - ${PYTHON_LINK_LIBRARY} - ) - - -if (WITH_TESTS) - enable_testing() - set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - add_subdirectory(test/gtest) - enable_testing() - include_directories(test/gtest/include .. ${CMAKE_CURRENT_BINARY_DIR}) - - - +install(TARGETS ${extension} LIBRARY DESTINATION "pdal") - PDAL_PYTHON_ADD_TEST(pdal_io_numpy_test - FILES - ./test/NumpyReaderTest.cpp - ./test/Support.cpp - ./plang/Invocation.cpp - ./plang/Environment.cpp - ./plang/Redirector.cpp - ./plang/Script.cpp - LINK_WITH - ${numpy_reader} - ${Python3_LIBRARIES} - ${PDAL_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - ) - PDAL_PYTHON_ADD_TEST(pdal_filters_python_test - FILES - ./test/PythonFilterTest.cpp - ./test/Support.cpp - ./plang/Invocation.cpp - ./plang/Environment.cpp - ./plang/Redirector.cpp - ./plang/Script.cpp - LINK_WITH - ${python_filter} - ${Python3_LIBRARIES} - ${PDAL_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - ) -endif (WITH_TESTS) diff --git a/pdal/macros.cmake b/pdal/macros.cmake index dd6f6d2f..7e11b1ee 100644 --- a/pdal/macros.cmake +++ b/pdal/macros.cmake @@ -91,7 +91,7 @@ macro(PDAL_PYTHON_ADD_PLUGIN _name _type _shortname) ${WINSOCK_LIBRARY} ) install(TARGETS ${${_name}} - LIBRARY DESTINATION ${PROJECT_NAME} + LIBRARY DESTINATION ${CMAKE_INSTALL_DIR}/pdal ) if (APPLE) set_target_properties(${${_name}} PROPERTIES diff --git a/setup.py b/setup.py index 5cac8dc4..f1c12656 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ def get_pdal_config(option): long_description = readme + '\n\n' + changes +# https://github.com/ktbarrett/hello/blob/master/CMakeLists.txt setup_args = dict( name = 'PDAL', @@ -89,10 +90,10 @@ def get_pdal_config(option): long_description = long_description, long_description_content_type = 'text/x-rst', test_suite = 'test', - cmake_source_dir = 'pdal', packages = [ 'pdal', ], + include_package_data = False, classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', From 03f4af3eeca3985b0f7829c4db2c3d072265e27c Mon Sep 17 00:00:00 2001 From: Tom Birch Date: Fri, 11 Dec 2020 01:44:58 +1100 Subject: [PATCH 002/161] Allow using "filters.python" in a python process on OS X (#76) Co-authored-by: Tom Birch --- CMakeLists.txt | 8 ++++++++ test/test_pypy.py | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 test/test_pypy.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 07e72a97..51ba0160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,14 @@ macro(PDAL_PYTHON_ADD_TEST _name) # endif() endmacro(PDAL_PYTHON_ADD_TEST) +if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # For newer versions of python (3.8+), C extensions don't link against + # libpython and instead get symbol definitions from the python interpreter + # executable. PDAL plugins need to link against libpython, but if a plugin + # is loaded inside a python process, it must resolve symbols from the python + # executable instead of libpython. Using flat namespace allows that. + set(PYTHON_LINK_LIBRARY ${PYTHON_LINK_LIBRARY} -Wl,-flat_namespace) +endif() PDAL_PYTHON_ADD_PLUGIN(numpy_reader reader numpy FILES diff --git a/test/test_pypy.py b/test/test_pypy.py new file mode 100644 index 00000000..67ef0ef9 --- /dev/null +++ b/test/test_pypy.py @@ -0,0 +1,26 @@ +import unittest +import json +import pdal +import numpy +import os + +DATADIRECTORY = "./test/data" + + +def a_filter(ins, outs): + return True + + +class TestPythonInPython(unittest.TestCase): + def test_pipeline_construction(self): + + pipeline = [ + os.path.join(DATADIRECTORY, "autzen-utm.las"), + { + "type": "filters.python", + "script": __file__, + "function": "a_filter", + "module": "anything", + }, + ] + pdal.Pipeline(json.dumps(pipeline)).execute() From 42197526699db979e5e151c55bd8bf4eeb300ef5 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 10 Dec 2020 08:57:52 -0600 Subject: [PATCH 003/161] cleanup and increment for 2.3.6 --- CMakeLists.txt | 53 ----------------- README.rst | 1 + pdal/__init__.py | 2 +- pdal/macros.cmake | 143 ---------------------------------------------- 4 files changed, 2 insertions(+), 197 deletions(-) delete mode 100644 pdal/macros.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 51ba0160..c8a533f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,16 +1,12 @@ cmake_minimum_required(VERSION 3.11.0) project(PDAL) -# macros for creating targets -#include(${PROJECT_SOURCE_DIR}/macros.cmake NO_POLICY_SCOPE) - set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) option(WITH_TESTS "Enable tests" OFF) set(CMAKE_BUILD_TYPE "Release") - # Python-finding settings set(Python3_FIND_STRATEGY "LOCATION") set(Python3_FIND_REGISTRY "LAST") @@ -26,56 +22,7 @@ if(SKBUILD) message(STATUS "The project is built using scikit-build") endif() -execute_process( - COMMAND - ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('Py_ENABLE_SHARED'))" - OUTPUT_VARIABLE Py_ENABLE_SHARED - OUTPUT_STRIP_TRAILING_WHITESPACE -) - -#execute_process( -# COMMAND -# ${Python3_EXECUTABLE} -c "from distutils import sysconfig; print(sysconfig.get_config_var('LDSHARED').split(' ', 1)[1])" -# OUTPUT_VARIABLE PYTHON_LDSHARED -# OUTPUT_STRIP_TRAILING_WHITESPACE -#) - -#if (NOT Py_ENABLE_SHARED) -# message(STATUS "Python ${Python3_EXECUTABLE} is statically linked") -# if (APPLE) -# # conda gives us -bundle, which isn't valid -# message(STATUS "Removing extra -bundle argument from sysconfig.get_config_var('LDSHARED')") -# string(REPLACE "-bundle" "" PYTHON_LDSHARED "${PYTHON_LDSHARED}") -# string(STRIP ${PYTHON_LDSHARED} PYTHON_LDSHARED) -# endif() -# # set(Python3_LIBRARIES ${PYTHON_LDSHARED}) -# message(STATUS "Setting Python3_LIBRARIES to '${Python3_LIBRARIES}' due to static Python") -#endif() - add_subdirectory(pdal) -#set(EXTENSION_SRC -# ./pdal/PyArray.cpp -# ./pdal/PyArray.hpp -# ./pdal/PyDimension.hpp -# ./pdal/PyPipeline.cpp -# ./pdal/PyPipeline.hpp) -# -#set(extension "libpdalpython") -#add_cython_target(${extension} "./pdal/libpdalpython.pyx" CXX PY3) -# -#add_library(${extension} MODULE ${EXTENSION_SRC} libpdalpython) -#target_include_directories( ${extension} -# PRIVATE -# . -# ${PDAL_INCLUDE_DIRS} -# ${Python3_INCLUDE_DIRS} -# ${Python3_NumPy_INCLUDE_DIRS}) -# -#target_link_libraries(${extension} ${PDAL_LIBRARIES}) -#python_extension_module(${extension}) -# -#install(TARGETS ${extension} LIBRARY DESTINATION "${PROJECT_NAME}/pdal") - # Taken and adapted from PDAL's cmake macros.cmake diff --git a/README.rst b/README.rst index 738c540d..5d5a4a9c 100644 --- a/README.rst +++ b/README.rst @@ -163,6 +163,7 @@ Requirements * PDAL 2.2+ * Python >=3.6 * Cython (eg :code:`pip install cython`) +* Numpy (eg :code:`pip install numpy`) * Packaging (eg :code:`pip install packaging`) * scikit-build (eg :code:`pip install scikit-build`) diff --git a/pdal/__init__.py b/pdal/__init__.py index 9430e234..07f396fb 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.3.5' +__version__='2.3.6' from .pipeline import Pipeline from .array import Array diff --git a/pdal/macros.cmake b/pdal/macros.cmake deleted file mode 100644 index 7e11b1ee..00000000 --- a/pdal/macros.cmake +++ /dev/null @@ -1,143 +0,0 @@ -# Taken and adapted from PDAL's cmake macros.cmake - -function(pdal_python_target_compile_settings target) - set_property(TARGET ${target} PROPERTY CXX_STANDARD 11) - set_property(TARGET ${target} PROPERTY CXX_STANDARD_REQUIRED TRUE) - target_compile_definitions(${target} PRIVATE - -DWIN32_LEAN_AND_MEAN) - if (MSVC) - # check for MSVC 8+ - if (NOT (MSVC_VERSION VERSION_LESS 1400)) - target_compile_definitions(${target} PRIVATE - -D_CRT_SECURE_NO_DEPRECATE - -D_CRT_SECURE_NO_WARNINGS - -D_CRT_NONSTDC_NO_WARNING - -D_SCL_SECURE_NO_WARNINGS - ) - target_compile_options(${target} PRIVATE - # Yes, we don't understand GCC pragmas - /wd4068 - # Nitro makes use of Exception Specifications, which results in - # numerous warnings when compiling in MSVC. We will ignore - # them for now. - /wd4290 - /wd4800 - # Windows warns about integer narrowing like crazy and it's - # annoying. In most cases the programmer knows what they're - # doing. A good static analysis tool would be better than - # turning this warning off. - /wd4267 - # Annoying warning about function hiding with virtual - # inheritance. - /wd4250 - # some templates don't return -# /wd4716 - # unwind semantics -# /wd4530 - # Standard C++-type exception handling. - /EHsc - ) - endif() - - endif() -endfunction() - - -############################################################################### -# Add a plugin target. -# _name The plugin name. -# ARGN : -# FILES the source files for the plugin -# LINK_WITH link plugin with libraries -# INCLUDES header directories -# -# The "generate_dimension_hpp" ensures that Dimension.hpp is built before -# attempting to build anything else in the "library". -# -# NOTE: _name is the name of a variable that will hold the plugin name -# when the macro completes -macro(PDAL_PYTHON_ADD_PLUGIN _name _type _shortname) - set(options) - set(oneValueArgs) - set(multiValueArgs FILES LINK_WITH INCLUDES SYSTEM_INCLUDES COMPILE_OPTIONS) - cmake_parse_arguments(PDAL_PYTHON_ADD_PLUGIN "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN}) - if(WIN32) - set(WINSOCK_LIBRARY ws2_32) - set(${_name} "libpdal_plugin_${_type}_${_shortname}") - else() - set(${_name} "pdal_plugin_${_type}_${_shortname}") - endif() - - - add_library(${${_name}} SHARED ${PDAL_PYTHON_ADD_PLUGIN_FILES}) - pdal_python_target_compile_settings(${${_name}}) - target_include_directories(${${_name}} PRIVATE - ${PROJECT_BINARY_DIR}/include - ${PDAL_INCLUDE_DIR} - ${PDAL_PYTHON_ADD_PLUGIN_INCLUDES} - ) - target_link_options(${${_name}} BEFORE PRIVATE ${PDAL_PYTHON_ADD_PLUGIN_COMPILE_OPTIONS}) - target_compile_definitions(${${_name}} PRIVATE - PDAL_PYTHON_LIBRARY="${PYTHON_LIBRARY}" PDAL_DLL_EXPORT) - target_compile_definitions(${${_name}} PRIVATE PDAL_DLL_EXPORT) - if (PDAL_PYTHON_ADD_PLUGIN_SYSTEM_INCLUDES) - target_include_directories(${${_name}} SYSTEM PRIVATE - ${PDAL_PYTHON_ADD_PLUGIN_SYSTEM_INCLUDES}) - endif() - target_link_libraries(${${_name}} - PRIVATE - ${PDAL_PYTHON_ADD_PLUGIN_LINK_WITH} - ${WINSOCK_LIBRARY} - ) - install(TARGETS ${${_name}} - LIBRARY DESTINATION ${CMAKE_INSTALL_DIR}/pdal - ) - if (APPLE) - set_target_properties(${${_name}} PROPERTIES - INSTALL_NAME_DIR "@rpath") - endif() -endmacro(PDAL_PYTHON_ADD_PLUGIN) - - -macro(PDAL_PYTHON_ADD_TEST _name) - set(options) - set(oneValueArgs) - set(multiValueArgs FILES LINK_WITH INCLUDES SYSTEM_INCLUDES) - cmake_parse_arguments(PDAL_PYTHON_ADD_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (WIN32) - set(WINSOCK_LIBRARY ws2_32) - endif() - add_executable(${_name} ${PDAL_PYTHON_ADD_TEST_FILES}) - - pdal_python_target_compile_settings(${_name}) - target_include_directories(${_name} PRIVATE - ${PDAL_PYTHON_ADD_TEST_INCLUDES}) - if (PDAL_PYTHON_ADD_TEST_SYSTEM_INCLUDES) - target_include_directories(${_name} SYSTEM PRIVATE - ${PDAL_PYTHON_ADD_TEST_SYSTEM_INCLUDES}) - endif() - set_property(TARGET ${_name} PROPERTY FOLDER "Tests") - target_link_libraries(${_name} - PRIVATE - ${PDAL_PYTHON_ADD_TEST_LINK_WITH} - gtest - ${WINSOCK_LIBRARY} - ) - target_compile_definitions(${_name} PRIVATE - PDAL_PYTHON_LIBRARY="${PYTHON_LIBRARY}") - add_test(NAME ${_name} - COMMAND - "${PROJECT_BINARY_DIR}/bin/${_name}" - WORKING_DIRECTORY - "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/..") - # Ensure plugins are loaded from build dir - # https://github.com/PDAL/PDAL/issues/840 -# if (WIN32) -# set_property(TEST ${_name} PROPERTY ENVIRONMENT -# "PDAL_DRIVER_PATH=${PROJECT_BINARY_DIR}/bin") -# else() -# set_property(TEST ${_name} PROPERTY ENVIRONMENT -# "PDAL_DRIVER_PATH=${PROJECT_BINARY_DIR}/lib") -# endif() -endmacro(PDAL_PYTHON_ADD_TEST) From 0004568fceb9375f6322f56dc1a47251b576991c Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 17 Jan 2021 21:03:25 +0200 Subject: [PATCH 004/161] Include CMakeLists.txt to the manifest (#80) --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6f1aec90..7233b061 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ exclude MANIFEST.in -include CHANGES.txt README.rst pyproject.toml tox.ini +include CHANGES.txt README.rst pyproject.toml tox.ini CMakeLists.txt include pdal/*.txt include pdal/*.cmake include test/* From 763d5c30df934fe7e30c01ef65a5a995ff286222 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 19 Jan 2021 09:56:33 -0600 Subject: [PATCH 005/161] bump to 2.3.7 for MANIFEST issue --- pdal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/__init__.py b/pdal/__init__.py index 07f396fb..d61c63e4 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.3.6' +__version__='2.3.7' from .pipeline import Pipeline from .array import Array From fea94f6e75b2d63477466ab66925106333cf80d0 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 29 Apr 2021 16:19:35 +0100 Subject: [PATCH 006/161] Add Meshes to the pdal API (#82) * first commit * pymesh typos * pymesh typos * pymesh bugs and added python defs * pypipeline bugs * Update PyPipeline.hpp * testing * testing * fixing the zero data bug * Update PyMesh.cpp * Update PyMesh.cpp * Update PyMesh.cpp * Update PyMesh.cpp * Update PyMesh.cpp * Update PyMesh.cpp * Added Meshio integration * meshio bug fixes * Delete .DS_Store * Delete .DS_Store * Update .gitignore * Delete .DS_Store * tests * test fix * Updates Docs * typos * typos * Update README.rst --- .gitignore | 1 + PKG-INFO | 42 +++++++- README.rst | 80 ++++++++++++++- pdal/CMakeLists.txt | 2 + pdal/PyMesh.cpp | 216 +++++++++++++++++++++++++++++++++++++++++ pdal/PyMesh.hpp | 103 ++++++++++++++++++++ pdal/PyPipeline.cpp | 19 ++++ pdal/PyPipeline.hpp | 3 +- pdal/libpdalpython.pyx | 21 ++++ pdal/pipeline.py | 23 +++++ test/data/mesh.json | 5 + test/test_pipeline.py | 27 ++++++ 12 files changed, 535 insertions(+), 7 deletions(-) create mode 100644 pdal/PyMesh.cpp create mode 100644 pdal/PyMesh.hpp create mode 100644 test/data/mesh.json diff --git a/.gitignore b/.gitignore index dbde1797..35f9baa3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ dist/* *.o *.so *.dylib +.DS_Store diff --git a/PKG-INFO b/PKG-INFO index 9dfc1d0b..43700f58 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -149,16 +149,48 @@ Description: =================================================================== count = p.execute() print (count) + Accessing Mesh Data + ................................................................................ + + Some PDAL stages (for instance ``filters.delaunay``) create TIN type mesh data. + + This data can be accessed in Python using the ``Pipeline.meshes`` property, which returns ``numpy.ndarray`` + of shape (1,n) where n is the number of Triangles in the mesh. If the PointView contains no mesh data, then n = 0. + + Each Triangle is a tuple ``(A,B,C)`` where A, B and C are indicees into the PointView for the point that is the vertex for the Triangle. + + Meshio Integration + ................................................................................ + + The meshes property provides the face data but is not easy to use as a mesh. Therefore, we have provided optional Integration + into the `Meshio `__ library. + + The ``pdal.Pipeline`` class provides the ``get_meshio(idx: int) -> meshio.Mesh`` method. This + method creates a `Mesh` object from the `PointView` array and mesh properties. + + .. note:: The meshio integration requires that meshio is installed (e.g. ``pip install meshio``). If it + is not, then the method fails with an informative RuntimeError. + + Simple use of the functionality could be as follows: + + -- code-block:: python + import pdal + + ... + pl = pdal.Pipeline(pipeline) + pl.execute() + mesh = pl.get_meshio(0) + mesh.write('test.obj') - .. _`Numpy`: http://www.numpy.org/ - .. _`schema`: http://www.pdal.io/dimensions.html - .. _`metadata`: http://www.pdal.io/development/metadata.html + .. _`Numpy`: http://www.numpy.org/ + .. _`schema`: http://www.pdal.io/dimensions.html + .. _`metadata`: http://www.pdal.io/development/metadata.html - .. image:: https://ci.appveyor.com/api/projects/status/of4kecyahpo8892d - :target: https://ci.appveyor.com/project/hobu/python/ + .. image:: https://ci.appveyor.com/api/projects/status/of4kecyahpo8892d + :target: https://ci.appveyor.com/project/hobu/python/ Requirements ================================================================================ diff --git a/README.rst b/README.rst index 5d5a4a9c..dce0df7b 100644 --- a/README.rst +++ b/README.rst @@ -62,7 +62,7 @@ sorts it by the ``X`` dimension: metadata = pipeline.metadata log = pipeline.log -Reading using Numpy Arrays +Reading using Numpy Arrays ................................................................................ The following more complex scenario demonstrates the full cycling between @@ -148,6 +148,84 @@ PDAL and Python: print (count) +Accessing Mesh Data +................................................................................ + +Some PDAL stages (for instance ``filters.delaunay``) create TIN type mesh data. + +This data can be accessed in Python using the ``Pipeline.meshes`` property, which returns a ``numpy.ndarray`` +of shape (1,n) where n is the number of Triangles in the mesh. + +If the PointView contains no mesh data, then n = 0. + +Each Triangle is a tuple ``(A,B,C)`` where A, B and C are indices into the PointView identifying the point that is the vertex for the Triangle. + +Meshio Integration +................................................................................ + +The meshes property provides the face data but is not easy to use as a mesh. Therefore, we have provided optional Integration +into the `Meshio `__ library. + +The ``pdal.Pipeline`` class provides the ``get_meshio(idx: int) -> meshio.Mesh`` method. This +method creates a `Mesh` object from the `PointView` array and mesh properties. + +.. note:: The meshio integration requires that meshio is installed (e.g. ``pip install meshio``). If it is not, then the method fails with an informative RuntimeError. + +Simple use of the functionality could be as follows: + +.. code-block:: python + + import pdal + + ... + pl = pdal.Pipeline(pipeline) + pl.execute() + + mesh = pl.get_meshio(0) + mesh.write('test.obj') + +Advanced Mesh Use Case +................................................................................ + +USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into tiles and store the tiles in PostGIS. + +.. note:: Like ``Pipeline.arrays``, ``Pipeline.meshes`` returns a list of ``numpy.ndarray`` to provide for the case where the output from a Pipeline is multiple PointViews + +(example using 1.2-with-color.las and not doing the ground classification for clarity) + +.. code-block:: python + + import pdal + import json + import psycopg2 + import io + + pipe = [ + '.../python/test/data/1.2-with-color.las', + {"type": "filters.splitter", "length": 1000}, + {"type": "filters.delaunay"} + ] + + pl = pdal.Pipeline(json.dumps(pipe)) + pl.execute() + + conn = psycopg(%CONNNECTION_STRING%) + buffer = io.StringIO + + for idx in range(len(pl.meshes)): + m = pl.get_meshio(idx) + if m: + m.write(buffer, file_format = "wkt") + with conn.cursor() as curr: + curr.execute( + "INSERT INTO %table-name% (mesh) VALUES (ST_GeomFromEWKT(%(ewkt)s)", + { "ewkt": buffer.getvalue()} + ) + + conn.commit() + conn.close() + buffer.close() + .. _`Numpy`: http://www.numpy.org/ diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt index 97bc051a..74b2d4dd 100644 --- a/pdal/CMakeLists.txt +++ b/pdal/CMakeLists.txt @@ -1,6 +1,8 @@ set(EXTENSION_SRC PyArray.cpp PyArray.hpp + PyMesh.cpp + PyMesh.hpp PyDimension.hpp PyPipeline.cpp PyPipeline.hpp) diff --git a/pdal/PyMesh.cpp b/pdal/PyMesh.cpp new file mode 100644 index 00000000..729e6a5c --- /dev/null +++ b/pdal/PyMesh.cpp @@ -0,0 +1,216 @@ +/****************************************************************************** +* Copyright (c) 2021, Runette Software Ltd (www.runette.co.uk) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "PyMesh.hpp" + +#include + + +namespace pdal +{ +namespace python +{ + +// Create new empty mesh +// +Mesh::Mesh() : m_mesh(nullptr) +{ + hasMesh = false; + if (_import_array() < 0) + throw pdal_error("Could not import numpy.core.multiarray."); +} + +Mesh::~Mesh() +{ + if (m_mesh) + Py_XDECREF((PyObject *)m_mesh); +} + +// Update from a PointView +// +void Mesh::update(PointViewPtr view) +{ + npy_intp size; + + if (m_mesh) + Py_XDECREF((PyObject *)m_mesh); + m_mesh = nullptr; // Just in case of an exception. + + TriangularMesh* mesh = view->mesh(); + if (mesh) + { + hasMesh = true; + size = mesh->size(); + } else { + hasMesh = false; + size = 0; + } + + PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); + if (!dtype_dict) + throw pdal_error("Unable to build numpy dtype " + "description dictionary"); + + PyArray_Descr *dtype = nullptr; + if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) + throw pdal_error("Unable to build numpy dtype"); + Py_XDECREF(dtype_dict); + + // This is a 1 x size array. + m_mesh = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); + + for (PointId idx = 0; idx < size; idx++) + { + char* p = (char *)PyArray_GETPTR1(m_mesh, idx); + const Triangle& t = (*mesh)[idx]; + uint32_t a = (uint32_t)t.m_a; + std::memcpy(p, &a, 4); + uint32_t b = (uint32_t)t.m_b; + std::memcpy(p + 4, &b, 4); + uint32_t c = (uint32_t)t.m_c; + std::memcpy(p + 8, &c, 4); + } +} + + + +PyArrayObject *Mesh::getPythonArray() const +{ + return m_mesh; +} + +PyObject* Mesh::buildNumpyDescription(PointViewPtr view) const +{ + // Build up a numpy dtype dictionary + // + // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', + // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], + // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', + // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', + // 'Classification', 'ScanAngleRank', 'UserData', + // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} + // + + Dimension::IdList dims = view->dims(); + + PyObject* dict = PyDict_New(); + PyObject* formats = PyList_New(3); + PyObject* titles = PyList_New(3); + + PyList_SetItem(titles, 0, PyUnicode_FromString("A")); + PyList_SetItem(formats, 0, PyUnicode_FromString("u4")); + PyList_SetItem(titles, 1, PyUnicode_FromString("B")); + PyList_SetItem(formats, 1, PyUnicode_FromString("u4")); + PyList_SetItem(titles, 2, PyUnicode_FromString("C")); + PyList_SetItem(formats, 2, PyUnicode_FromString("u4")); + + + PyDict_SetItemString(dict, "names", titles); + PyDict_SetItemString(dict, "formats", formats); + + return dict; +} + +bool Mesh::rowMajor() const +{ + return m_rowMajor; +} + +Mesh::Shape Mesh::shape() const +{ + return m_shape; +} + + +MeshIter& Mesh::iterator() +{ + MeshIter *it = new MeshIter(*this); + m_iterators.push_back(std::unique_ptr(it)); + return *it; +} + +MeshIter::MeshIter(Mesh& mesh) +{ + m_iter = NpyIter_New(mesh.getPythonArray(), + NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, + NPY_KEEPORDER, NPY_NO_CASTING, NULL); + if (!m_iter) + throw pdal_error("Unable to create numpy iterator."); + + char *itererr; + m_iterNext = NpyIter_GetIterNext(m_iter, &itererr); + if (!m_iterNext) + { + NpyIter_Deallocate(m_iter); + throw pdal_error(std::string("Unable to create numpy iterator: ") + + itererr); + } + m_data = NpyIter_GetDataPtrArray(m_iter); + m_stride = NpyIter_GetInnerStrideArray(m_iter); + m_size = NpyIter_GetInnerLoopSizePtr(m_iter); + m_done = false; +} + +MeshIter::~MeshIter() +{ + NpyIter_Deallocate(m_iter); +} + +MeshIter& MeshIter::operator++() +{ + if (m_done) + return *this; + + if (--(*m_size)) + *m_data += *m_stride; + else if (!m_iterNext(m_iter)) + m_done = true; + return *this; +} + +MeshIter::operator bool () const +{ + return !m_done; +} + +char * MeshIter::operator * () const +{ + return *m_data; +} + + +} // namespace python +} // namespace pdal + diff --git a/pdal/PyMesh.hpp b/pdal/PyMesh.hpp new file mode 100644 index 00000000..6b8b027d --- /dev/null +++ b/pdal/PyMesh.hpp @@ -0,0 +1,103 @@ +/****************************************************************************** +* Copyright (c) 2021, Runette Software Ltd (www.runette.co.uk) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + + +#include +#include + +#include + +namespace pdal +{ +namespace python +{ +class MeshIter; + +class PDAL_DLL Mesh +{ +public: + using Shape = std::array; + + bool hasMesh; + + Mesh(); + + ~Mesh(); + + void update(PointViewPtr view); + PyArrayObject *getPythonArray() const; + + bool rowMajor() const; + Shape shape() const; + MeshIter& iterator(); + + +private: + inline PyObject* buildNumpyDescription(PointViewPtr view) const; + + PyArrayObject* m_mesh; + + Mesh& operator=(Mesh const& rhs); + bool m_rowMajor; + Shape m_shape {}; + std::vector> m_iterators; +}; + +class MeshIter +{ +public: + MeshIter(const MeshIter&) = delete; + MeshIter() = delete; + + MeshIter(Mesh& mesh); + ~MeshIter(); + + MeshIter& operator++(); + operator bool () const; + char *operator * () const; + +private: + NpyIter *m_iter; + NpyIter_IterNextFunc *m_iterNext; + char **m_data; + npy_intp *m_size; + npy_intp *m_stride; + bool m_done; +}; + +} // namespace python +} // namespace pdal + diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 139bc82e..215652f6 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -45,6 +45,7 @@ #include #include "PyArray.hpp" +#include "PyMesh.hpp" namespace pdal { @@ -169,6 +170,24 @@ std::vector Pipeline::getArrays() const return output; } +std::vector Pipeline::getMeshes() const +{ + std::vector output; + + if (!m_executor->executed()) + throw python_error("call execute() before fetching the mesh"); + + const PointViewSet& pvset = m_executor->getManagerConst().views(); + + for (auto i: pvset) + { + Mesh *mesh = new python::Mesh; + mesh->update(i); + output.push_back(mesh); + } + return output; +} + } // namespace python } // namespace pdal diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index a1ddc30b..ef186d6a 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -49,6 +49,7 @@ namespace python { class Array; +class Mesh; class python_error : public std::runtime_error { @@ -84,7 +85,7 @@ class Pipeline return m_executor->getLog(); } std::vector getArrays() const; - + std::vector getMeshes() const; void setLogLevel(int level); int getLogLevel() const; diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 727139d8..5d6b8328 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -41,6 +41,11 @@ cdef extern from "PyArray.hpp" namespace "pdal::python": Array(np.ndarray) except + void *getPythonArray() except+ +cdef extern from "PyMesh.hpp" namespace "pdal::python": + cdef cppclass Mesh: + Mesh(np.ndarray) except + + void *getPythonArray() except + + cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef cppclass Pipeline: Pipeline(const char* ) except + @@ -52,6 +57,7 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": string getSchema() except + string getLog() except + vector[Array*] getArrays() except + + vector[Mesh*] getMeshes() except + int getLogLevel() void setLogLevel(int) @@ -152,6 +158,21 @@ cdef class PyPipeline: inc(it) return output + property meshes: + + def __get__(self): + v = self.thisptr.getMeshes() + output = [] + cdef vector[Mesh *].iterator it = v.begin() + cdef Mesh* m + while it != v.end(): + ptr = deref(it) + m = ptr#.get() + o = m.getPythonArray() + output.append(o) + del ptr + inc(it) + return output def execute(self): if not self.thisptr: diff --git a/pdal/pipeline.py b/pdal/pipeline.py index fca579e9..e9d502a0 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -45,3 +45,26 @@ def validate(self): def get_arrays(self): return self.p.arrays arrays = property(get_arrays) + + def get_meshes(self): + return self.p.meshes + meshes = property(get_meshes) + + def get_meshio(self, idx: int): + try: + from meshio import Mesh + except ModuleNotFoundError: + raise RuntimeError( + "The get_meshio function can only be used if you have installed meshio. Try pip install meshio") + array = self.arrays[idx] + mesh = self.meshes[idx] + if len(mesh) == 0: + return None + return Mesh( + np.stack((array['X'], array['Y'], array['Z']), 1), + [( + "triangle", + np.stack((mesh['A'], mesh['B'], mesh['C']), 1) + ) + ] + ) \ No newline at end of file diff --git a/test/data/mesh.json b/test/data/mesh.json new file mode 100644 index 00000000..e44d9356 --- /dev/null +++ b/test/data/mesh.json @@ -0,0 +1,5 @@ +[ + "test/data/1.2-with-color.las", + {"type": "filters.splitter", "length": 1000}, + {"type": "filters.delaunay"} +] \ No newline at end of file diff --git a/test/test_pipeline.py b/test/test_pipeline.py index c7560ace..eb73d346 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -231,6 +231,33 @@ def test_fetch_dimensions(self): self.assertLess(len(dims), 100) self.assertGreater(len(dims), 71) +class TestMesh(PDALTest): + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), + "missing test data") + def test_no_execute(self): + """Does fetching meshes without executing throw an exception""" + json = self.fetch_json('sort.json') + r = pdal.Pipeline(json) + with self.assertRaises(RuntimeError): + r.meshes + + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'mesh.json')), + "missing test data") + def test_mesh(self): + """Can we fetch PDAL face data as a numpy array""" + json = self.fetch_json('mesh.json') + r = pdal.Pipeline(json) + r.validate() + points = r.execute() + self.assertEqual(points, 1065) + meshes = r.meshes + self.assertEqual(len(meshes), 24) + + m = meshes[0] + self.assertEqual(str(m.dtype), "[('A', ' Date: Thu, 29 Apr 2021 10:23:37 -0500 Subject: [PATCH 007/161] bump to 2.4.0 for mesh support --- pdal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/__init__.py b/pdal/__init__.py index d61c63e4..a687ad30 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.3.7' +__version__='2.4.0' from .pipeline import Pipeline from .array import Array From d39a6c9c656f6d9dac1111505b8808f8a6f81844 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 24 Jun 2021 11:41:37 -0400 Subject: [PATCH 008/161] Use out_metadata for stage metadata. --- pdal/plang/Environment.cpp | 90 ++++++++++++++++++-------------------- pdal/plang/Invocation.cpp | 7 ++- 2 files changed, 45 insertions(+), 52 deletions(-) diff --git a/pdal/plang/Environment.cpp b/pdal/plang/Environment.cpp index bf813b9f..251b0394 100644 --- a/pdal/plang/Environment.cpp +++ b/pdal/plang/Environment.cpp @@ -79,6 +79,29 @@ static void loadPython() } #endif +namespace +{ + +std::string readPythonString(PyObject* dict, const std::string& key) +{ + std::string s; + + PyObject* o = PyDict_GetItemString(dict, key.c_str()); + if (!o) + return s; + + PyObject* r = PyObject_Str(o); + if (r) + { + Py_ssize_t size; + s = PyUnicode_AsUTF8AndSize(r, &size); + } + + return s; +} + +} // unnamed namespace + // http://www.linuxjournal.com/article/3641 // http://www.codeproject.com/Articles/11805/Embedding-Python-in-C-C-Part-I // http://stackoverflow.com/questions/6596016/python-threads-in-c @@ -231,21 +254,6 @@ std::string getTraceback() // Returns a new reference. PyObject *fromMetadata(MetadataNode m) { - std::string name = m.name(); - std::string value = m.value(); - std::string type = m.type(); - std::string description = m.description(); - - MetadataNodeList children = m.children(); - PyObject *submeta(0); - if (children.size()) - { - submeta = PyList_New(0); - for (MetadataNode& child : children) - PyList_Append(submeta, fromMetadata(child)); - } - PyObject *data = PyDict_New(); - auto getString = [](const std::string& s) { PyObject *o = PyUnicode_FromString(s.data()); @@ -254,68 +262,54 @@ PyObject *fromMetadata(MetadataNode m) return o; }; - PyDict_SetItemString(data, "name", getString(name)); - PyDict_SetItemString(data, "value", getString(value)); - PyDict_SetItemString(data, "type", getString(value)); - PyDict_SetItemString(data, "description", getString(description)); + PyObject *data = PyDict_New(); + PyDict_SetItemString(data, "name", getString(m.name())); + PyDict_SetItemString(data, "value", getString(m.value())); + PyDict_SetItemString(data, "type", getString(m.type())); + PyDict_SetItemString(data, "description", getString(m.description())); + + MetadataNodeList children = m.children(); if (children.size()) + { + PyObject *submeta = PyList_New(0); + for (MetadataNode& child : children) + PyList_Append(submeta, fromMetadata(child)); PyDict_SetItemString(data, "children", submeta); + } + return data; } -std::string readPythonString(PyObject* dict, const std::string& key) -{ - std::stringstream ss; - PyObject* o = PyDict_GetItemString(dict, key.c_str()); - if (!o) - { - std::stringstream oss; - oss << "Unable to get dictionary item '" << key << "'"; - throw pdal_error(oss.str()); - } - - PyObject* r = PyObject_Str(o); - if (!r) - throw pdal::pdal_error("unable to get repr in readPythonString"); - Py_ssize_t size; - const char *d = PyUnicode_AsUTF8AndSize(r, &size); - ss << d; - - return ss.str(); -} void addMetadata(PyObject *dict, MetadataNode m) { if (!dict) - { return; - } if (!PyDict_Check(dict)) - throw pdal::pdal_error("'metadata' member must be a dictionary!"); + throw pdal::pdal_error("Output metadata must be in a dictionary."); std::string name = readPythonString(dict, "name"); std::string value = readPythonString(dict, "value"); - std::string type = readPythonString(dict, "type"); + std::string description = readPythonString(dict, "description"); + if (name.empty()) + return; if (type.empty()) type = Metadata::inferType(value); - std::string description = readPythonString(dict, "description"); - + m = m.addWithType(name, value, type, description); PyObject *submeta = PyDict_GetItemString(dict, "children"); if (submeta) { if (!PyList_Check(submeta)) - throw pdal::pdal_error("'children' metadata member must be a list!"); - + throw pdal::pdal_error("Ouput metadata 'children' must be a list."); for (Py_ssize_t i = 0; i < PyList_Size(submeta); ++i) { PyObject* p = PyList_GetItem(submeta, i); addMetadata(p, m); } - MetadataNode child = m.addWithType(name, value, type, description); } } diff --git a/pdal/plang/Invocation.cpp b/pdal/plang/Invocation.cpp index c74f9245..8268ae9c 100644 --- a/pdal/plang/Invocation.cpp +++ b/pdal/plang/Invocation.cpp @@ -384,7 +384,7 @@ PyObject *Invocation::prepareData(PointViewPtr& view) MetadataNode layoutMeta = view->layout()->toMetadata(); MetadataNode srsMeta = view->spatialReference().toMetadata(); - addGlobalObject(m_module, plang::fromMetadata(m_inputMetadata), "metadata"); + addGlobalObject(m_module, plang::fromMetadata(m_inputMetadata), "stage_metadata"); addGlobalObject(m_module, getPyJSON(m_pdalargs), "pdalargs"); addGlobalObject(m_module, getPyJSON(Utils::toJSON(layoutMeta)), "schema"); addGlobalObject(m_module, getPyJSON(Utils::toJSON(srsMeta)), @@ -473,11 +473,10 @@ void Invocation::extractData(PointViewPtr& view, PyObject *arrays) void Invocation::extractMetadata(MetadataNode stageMetadata) { - PyObject *key = PyUnicode_FromString("metadata"); + PyObject *key = PyUnicode_FromString("out_metadata"); PyObject *dictionary = PyModule_GetDict(m_module); PyObject *pyMeta = PyDict_GetItem(dictionary, key); - if (pyMeta) - addMetadata(pyMeta, stageMetadata); + addMetadata(pyMeta, stageMetadata); Py_DECREF(key); } From 9c8541526dac1feba390744a6f87bc12d68ddf1d Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 24 Jun 2021 11:46:08 -0400 Subject: [PATCH 009/161] Revert input metadata object name. --- pdal/plang/Invocation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/plang/Invocation.cpp b/pdal/plang/Invocation.cpp index 8268ae9c..380ac132 100644 --- a/pdal/plang/Invocation.cpp +++ b/pdal/plang/Invocation.cpp @@ -384,7 +384,7 @@ PyObject *Invocation::prepareData(PointViewPtr& view) MetadataNode layoutMeta = view->layout()->toMetadata(); MetadataNode srsMeta = view->spatialReference().toMetadata(); - addGlobalObject(m_module, plang::fromMetadata(m_inputMetadata), "stage_metadata"); + addGlobalObject(m_module, plang::fromMetadata(m_inputMetadata), "metadata"); addGlobalObject(m_module, getPyJSON(m_pdalargs), "pdalargs"); addGlobalObject(m_module, getPyJSON(Utils::toJSON(layoutMeta)), "schema"); addGlobalObject(m_module, getPyJSON(Utils::toJSON(srsMeta)), From 72c5a5ddfa43daa4ba9c85e48df1ffb0c5c5e1df Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 24 Jun 2021 12:07:07 -0400 Subject: [PATCH 010/161] Change output metadata variable name. --- pdal/test/PythonFilterTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/test/PythonFilterTest.cpp b/pdal/test/PythonFilterTest.cpp index 1db5b8ee..d3d40976 100644 --- a/pdal/test/PythonFilterTest.cpp +++ b/pdal/test/PythonFilterTest.cpp @@ -225,7 +225,7 @@ TEST_F(PythonFilterTest, metadata) "def myfunc(ins,outs):\n" " global metadata\n" " #print('before', globals(), file=sys.stderr,)\n" - " metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'filters.python', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'readers.faux', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}\n" + " out_metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'filters.python', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'readers.faux', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}\n" " # print ('schema', schema, file=sys.stderr,)\n" " return True\n" ); From 77358e6690cc1fc0f9aab1fd6b9b1dcdf9c05db2 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 24 Jun 2021 12:36:44 -0400 Subject: [PATCH 011/161] Test. --- pdal/test/PythonFilterTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pdal/test/PythonFilterTest.cpp b/pdal/test/PythonFilterTest.cpp index d3d40976..ffbbcc91 100644 --- a/pdal/test/PythonFilterTest.cpp +++ b/pdal/test/PythonFilterTest.cpp @@ -225,7 +225,7 @@ TEST_F(PythonFilterTest, metadata) "def myfunc(ins,outs):\n" " global metadata\n" " #print('before', globals(), file=sys.stderr,)\n" - " out_metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'filters.python', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'readers.faux', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}\n" + " out_metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'somechildren', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'otherchildren', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}\n" " # print ('schema', schema, file=sys.stderr,)\n" " return True\n" ); @@ -248,7 +248,7 @@ TEST_F(PythonFilterTest, metadata) PointLayoutPtr layout(table.layout()); MetadataNode m = table.metadata(); - m = m.findChild("filters.python"); + m = m.findChild("somechildren"); MetadataNodeList l = m.children(); EXPECT_EQ(l.size(), 3u); EXPECT_EQ(l[0].name(), "filters.python"); From ef9de96865beb0b55939d01a72e848c87f90358b Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 24 Jun 2021 12:53:15 -0400 Subject: [PATCH 012/161] Test. --- pdal/test/PythonFilterTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/pdal/test/PythonFilterTest.cpp b/pdal/test/PythonFilterTest.cpp index ffbbcc91..ccb6ff75 100644 --- a/pdal/test/PythonFilterTest.cpp +++ b/pdal/test/PythonFilterTest.cpp @@ -249,6 +249,7 @@ TEST_F(PythonFilterTest, metadata) PointLayoutPtr layout(table.layout()); MetadataNode m = table.metadata(); m = m.findChild("somechildren"); + Utils::toJSON(m, std::cerr); MetadataNodeList l = m.children(); EXPECT_EQ(l.size(), 3u); EXPECT_EQ(l[0].name(), "filters.python"); From 6413e09060d3c1b30ae166fbf0aae890b65f5f21 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 25 Jun 2021 08:44:59 -0400 Subject: [PATCH 013/161] Fix metadata test. --- pdal/plang/Environment.cpp | 13 +++++++++---- pdal/test/PythonFilterTest.cpp | 14 +++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pdal/plang/Environment.cpp b/pdal/plang/Environment.cpp index 251b0394..63f92210 100644 --- a/pdal/plang/Environment.cpp +++ b/pdal/plang/Environment.cpp @@ -299,16 +299,21 @@ void addMetadata(PyObject *dict, MetadataNode m) if (type.empty()) type = Metadata::inferType(value); - m = m.addWithType(name, value, type, description); + m.addWithType(name, value, type, description); PyObject *submeta = PyDict_GetItemString(dict, "children"); if (submeta) { if (!PyList_Check(submeta)) throw pdal::pdal_error("Ouput metadata 'children' must be a list."); - for (Py_ssize_t i = 0; i < PyList_Size(submeta); ++i) + Py_ssize_t size = PyList_Size(submeta); + if (size) { - PyObject* p = PyList_GetItem(submeta, i); - addMetadata(p, m); + m = m.add("children"); + for (Py_ssize_t i = 0; i < PyList_Size(submeta); ++i) + { + PyObject* p = PyList_GetItem(submeta, i); + addMetadata(p, m); + } } } } diff --git a/pdal/test/PythonFilterTest.cpp b/pdal/test/PythonFilterTest.cpp index ccb6ff75..d114ec93 100644 --- a/pdal/test/PythonFilterTest.cpp +++ b/pdal/test/PythonFilterTest.cpp @@ -223,7 +223,7 @@ TEST_F(PythonFilterTest, metadata) "import sys\n" "import redirector\n" "def myfunc(ins,outs):\n" - " global metadata\n" + " global out_metadata\n" " #print('before', globals(), file=sys.stderr,)\n" " out_metadata = {'name': 'root', 'value': 'a string', 'type': 'string', 'description': 'a description', 'children': [{'name': 'somechildren', 'value': 52, 'type': 'integer', 'description': 'a filter description', 'children': []}, {'name': 'otherchildren', 'value': 'another string', 'type': 'string', 'description': 'a reader description', 'children': []}]}\n" " # print ('schema', schema, file=sys.stderr,)\n" @@ -248,13 +248,13 @@ TEST_F(PythonFilterTest, metadata) PointLayoutPtr layout(table.layout()); MetadataNode m = table.metadata(); - m = m.findChild("somechildren"); - Utils::toJSON(m, std::cerr); + m = m.findChild("filters.python"); MetadataNodeList l = m.children(); - EXPECT_EQ(l.size(), 3u); - EXPECT_EQ(l[0].name(), "filters.python"); - EXPECT_EQ(l[0].value(), "52"); - EXPECT_EQ(l[0].description(), "a filter description"); + EXPECT_EQ(l.size(), 2u); + m = m.findChild("children"); + EXPECT_EQ(m.children().size(), 2u); + m = m.findChild("somechildren"); + EXPECT_EQ(m.value(), "52"); } TEST_F(PythonFilterTest, pdalargs) From a791ce3a4fab7ea04eb3edb63d1d74a0a15fe202 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 25 Jun 2021 09:13:05 -0400 Subject: [PATCH 014/161] More dimensions. --- test/test_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index eb73d346..0113428c 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -228,7 +228,7 @@ class TestDimensions(PDALTest): def test_fetch_dimensions(self): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - self.assertLess(len(dims), 100) + self.assertLess(len(dims), 120) self.assertGreater(len(dims), 71) class TestMesh(PDALTest): From f726dc4c8ba1d656847d0ca27e617972dda188c1 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 25 Jun 2021 10:00:41 -0400 Subject: [PATCH 015/161] Don't mess with string size if we don't care. --- pdal/plang/Environment.cpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/pdal/plang/Environment.cpp b/pdal/plang/Environment.cpp index 63f92210..02968823 100644 --- a/pdal/plang/Environment.cpp +++ b/pdal/plang/Environment.cpp @@ -92,10 +92,7 @@ std::string readPythonString(PyObject* dict, const std::string& key) PyObject* r = PyObject_Str(o); if (r) - { - Py_ssize_t size; - s = PyUnicode_AsUTF8AndSize(r, &size); - } + s = PyUnicode_AsUTF8AndSize(r, NULL); return s; } @@ -222,9 +219,9 @@ std::string getTraceback() PyObject* r = PyObject_Repr(l); if (!r) throw pdal::pdal_error("unable to get repr in getTraceback"); - Py_ssize_t size; - const char *d = PyUnicode_AsUTF8AndSize(r, &size); - mssg << d; + const char *d = PyUnicode_AsUTF8AndSize(r, NULL); + if (d) + mssg << d; } // clean up @@ -236,9 +233,9 @@ std::string getTraceback() PyObject* r = PyObject_Repr(value); if (!r) throw pdal::pdal_error("couldn't make string representation of traceback value"); - Py_ssize_t size; - const char *d = PyUnicode_AsUTF8AndSize(r, &size); - mssg << d; + const char *d = PyUnicode_AsUTF8AndSize(r, NULL); + if (d) + mssg << d; } else mssg << "unknown error that we are unable to get a traceback for." From ca02b0385929fefa2be02c9ccd5df25c9e77b0d8 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Fri, 25 Jun 2021 10:03:53 -0400 Subject: [PATCH 016/161] Simplify. --- pdal/plang/Environment.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pdal/plang/Environment.cpp b/pdal/plang/Environment.cpp index 02968823..fbded805 100644 --- a/pdal/plang/Environment.cpp +++ b/pdal/plang/Environment.cpp @@ -87,13 +87,12 @@ std::string readPythonString(PyObject* dict, const std::string& key) std::string s; PyObject* o = PyDict_GetItemString(dict, key.c_str()); - if (!o) - return s; - - PyObject* r = PyObject_Str(o); - if (r) - s = PyUnicode_AsUTF8AndSize(r, NULL); - + if (o) + { + PyObject* r = PyObject_Str(o); + if (r) + s = PyUnicode_AsUTF8AndSize(r, NULL); + } return s; } From 7a2c36239edd1ea8be34b5dfaa252b38522c3efe Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 28 Jun 2021 11:26:53 -0500 Subject: [PATCH 017/161] bump version --- pdal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/__init__.py b/pdal/__init__.py index a687ad30..497b4636 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.4.0' +__version__='2.4.2' from .pipeline import Pipeline from .array import Array From 8f790d302d7b6f81b7b14ac1d14909c7d35763fb Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 3 Oct 2021 21:40:37 +0300 Subject: [PATCH 018/161] Minimize CI Conda dependencies --- .github/workflows/build.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59ec8882..804e1acf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,18 +40,14 @@ jobs: - name: Dependencies shell: bash -l {0} run: | - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} compilers cython ninja pdal pytest conda-build -y + conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest - name: Install shell: bash -l {0} run: | python setup.py install -- -G "Ninja" -DWITH_TESTS=ON - pdal --drivers - -# pip install . --build-option="-DWITH_TESTS=ON" - - name: Test shell: bash -l {0} run: | @@ -63,7 +59,6 @@ jobs: $SKPATH/pdal_io_numpy_test py.test - windows: name: ${{ matrix.os }} - ${{ matrix.python-version }} @@ -82,18 +77,15 @@ jobs: auto-update-conda: true python-version: ${{ matrix.python-version }} - - name: Dependencies shell: cmd /C CALL "{0}" run: | - call conda activate test - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} compilers cython ninja pdal pytest conda-build -y + conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest - name: Install shell: cmd /C CALL "{0}" run: | - call conda activate test where python python setup.py develop -- -G "Ninja" -DWITH_TESTS=ON @@ -133,7 +125,7 @@ jobs: - name: Dependencies shell: bash -l {0} run: | - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} compilers cython ninja pdal pytest conda-build -y + conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest - name: sdist shell: bash -l {0} From 726c57399af19dff0cfe61aa213ab30f987afd86 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 7 Sep 2021 00:47:54 +0300 Subject: [PATCH 019/161] Cleanup setup.py --- setup.py | 131 ++++++++++++++----------------------------------------- 1 file changed, 33 insertions(+), 98 deletions(-) diff --git a/setup.py b/setup.py index f1c12656..d3aba0fd 100644 --- a/setup.py +++ b/setup.py @@ -1,111 +1,46 @@ -#!/usr/bin/env python - -# Stolen from Shapely's setup.py -# Two environment variables influence this script. -# -# PDAL_LIBRARY_PATH: a path to a PDAL C++ shared library. -# -# PDAL_CONFIG: the path to a pdal-config program that points to PDAL version, -# headers, and libraries. -# -# NB: within this setup scripts, software versions are evaluated according -# to https://www.python.org/dev/peps/pep-0440/. - -import logging -import os -import platform -import sys -import numpy -import glob -import sysconfig -import setuptools from skbuild import setup -from packaging.version import Version - - - -def get_pdal_config(option): - '''Get configuration option from the `pdal-config` development utility - - This code was adapted from Shapely's geos-config stuff - ''' - import subprocess - pdal_config = os.environ.get('PDAL_CONFIG','pdal-config') - if not pdal_config or not isinstance(pdal_config, str): - raise OSError('Path to pdal-config is not set') - try: - stdout, stderr = subprocess.Popen( - [pdal_config, option], - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - except OSError as ex: - # e.g., [Errno 2] No such file or directory - raise OSError( - 'Could not find pdal-config %r: %s' % (pdal_config, ex)) - if stderr and not stdout: - raise ValueError(stderr.strip()) - if sys.version_info[0] >= 3: - result = stdout.decode('ascii').strip() - else: - result = stdout.strip() - return result # Get the version from the pdal module -module_version = None -with open('pdal/__init__.py', 'r') as fp: +with open("pdal/__init__.py", "r") as fp: for line in fp: if line.startswith("__version__"): - module_version = Version(line.split("=")[1].strip().strip("\"'")) + version = line.split("=")[1].strip().strip("\"'") break + else: + raise ValueError("Could not determine Python package version") -if not module_version: - raise ValueError("Could not determine Python package version") - -# Handle UTF-8 encoding of certain text files. -open_kwds = {} -if sys.version_info >= (3,): - open_kwds['encoding'] = 'utf-8' - -with open('README.rst', 'r', **open_kwds) as fp: +with open("README.rst", "r", encoding="utf-8") as fp: readme = fp.read() -with open('CHANGES.txt', 'r', **open_kwds) as fp: +with open("CHANGES.txt", "r", encoding="utf-8") as fp: changes = fp.read() -long_description = readme + '\n\n' + changes - -# https://github.com/ktbarrett/hello/blob/master/CMakeLists.txt - -setup_args = dict( - name = 'PDAL', - version = str(module_version), - requires = ['Python (>=3.0)', 'Numpy'], - description = 'Point cloud data processing', - license = 'BSD', - keywords = 'point cloud spatial', - author = 'Howard Butler', - author_email = 'howard@hobu.co', - maintainer = 'Howard Butler', - maintainer_email = 'howard@hobu.co', - url = 'https://pdal.io', - long_description = long_description, - long_description_content_type = 'text/x-rst', - test_suite = 'test', - packages = [ - 'pdal', +setup( + name="PDAL", + version=version, + requires=["Python (>=3.0)", "Numpy"], + description="Point cloud data processing", + license="BSD", + keywords="point cloud spatial", + author="Howard Butler", + author_email="howard@hobu.co", + maintainer="Howard Butler", + maintainer_email="howard@hobu.co", + url="https://pdal.io", + long_description=readme + "\n\n" + changes, + long_description_content_type="text/x-rst", + test_suite="test", + packages=["pdal"], + include_package_data=False, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Topic :: Scientific/Engineering :: GIS", ], - include_package_data = False, - classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Topic :: Scientific/Engineering :: GIS', - ], - ) -output = setup(**setup_args) - From 56871abfa779008d456818ecd74de5118f24acfe Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 8 Sep 2021 12:51:24 +0300 Subject: [PATCH 020/161] Drop Array and PyArray wrapped classes --- pdal/__init__.py | 22 +++++++++++++++++----- pdal/array.py | 8 -------- pdal/dimension.py | 6 ------ pdal/libpdalpython.pyx | 7 ------- 4 files changed, 17 insertions(+), 26 deletions(-) delete mode 100644 pdal/array.py delete mode 100644 pdal/dimension.py diff --git a/pdal/__init__.py b/pdal/__init__.py index 497b4636..49e9f603 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,10 +1,21 @@ -__version__='2.4.2' - +from .libpdalpython import ( + getDebugInformation, + getDimensions, + getPluginInstallPath, + getSha1, + getVersionMajor, + getVersionMinor, + getVersionPatch, + getVersionString, +) from .pipeline import Pipeline -from .array import Array -from .dimension import dimensions -from pdal.libpdalpython import getVersionString, getVersionMajor, getVersionMinor, getVersionPatch, getSha1, getDebugInformation, getPluginInstallPath + +__version__ = "2.4.2" + +dimensions = getDimensions() +del getDimensions + class Info(object): version = getVersionString() @@ -15,4 +26,5 @@ class Info(object): sha1 = getSha1() plugin = getPluginInstallPath() + info = Info() diff --git a/pdal/array.py b/pdal/array.py deleted file mode 100644 index d9859278..00000000 --- a/pdal/array.py +++ /dev/null @@ -1,8 +0,0 @@ -import numpy as np -from pdal import libpdalpython - -class Array(object): - """A Numpy Array that can speak PDAL""" - - def __init__(self, data): - self.p = libpdalpython.PyArray(data) diff --git a/pdal/dimension.py b/pdal/dimension.py deleted file mode 100644 index 867609b1..00000000 --- a/pdal/dimension.py +++ /dev/null @@ -1,6 +0,0 @@ -import numpy as np -from pdal import libpdalpython -from pdal.libpdalpython import getDimensions - -dimensions = getDimensions() - diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 5d6b8328..2281190e 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -61,13 +61,6 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": int getLogLevel() void setLogLevel(int) -cdef class PyArray: - cdef Array *thisptr - def __cinit__(self, np.ndarray array): - self.thisptr = new Array(array) - def __dealloc__(self): - del self.thisptr - cdef extern from "PyDimension.hpp": ctypedef struct Dimension: string name; From a692f0653b71a1f3e83be16feb26e5378ed7f4cc Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 8 Sep 2021 13:02:33 +0300 Subject: [PATCH 021/161] Drop all the pdal.get* functions, define pdal.info as a SimpleNamespace instance --- pdal/__init__.py | 28 +++------------------------- pdal/libpdalpython.pyx | 31 ++++++++++++++----------------- 2 files changed, 17 insertions(+), 42 deletions(-) diff --git a/pdal/__init__.py b/pdal/__init__.py index 49e9f603..5109f32c 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,30 +1,8 @@ -from .libpdalpython import ( - getDebugInformation, - getDimensions, - getPluginInstallPath, - getSha1, - getVersionMajor, - getVersionMinor, - getVersionPatch, - getVersionString, -) +from .libpdalpython import getDimensions, getInfo from .pipeline import Pipeline - __version__ = "2.4.2" dimensions = getDimensions() -del getDimensions - - -class Info(object): - version = getVersionString() - major = getVersionMajor() - minor = getVersionMinor() - patch = getVersionPatch() - debug = getDebugInformation() - sha1 = getSha1() - plugin = getPluginInstallPath() - - -info = Info() +info = getInfo() +del getDimensions, getInfo diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 2281190e..cafd6ae1 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -3,13 +3,12 @@ from libcpp.vector cimport vector from libcpp.string cimport string -from libc.stdint cimport uint32_t, int64_t +from libc.stdint cimport int64_t from libcpp cimport bool -from cpython.version cimport PY_MAJOR_VERSION +from types import SimpleNamespace cimport numpy as np np.import_array() -from cpython cimport PyObject, Py_INCREF from cython.operator cimport dereference as deref, preincrement as inc cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": @@ -21,20 +20,18 @@ cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": cdef string pluginInstallPath() except+ cdef string versionString() except+ -def getVersionString(): - return versionString() -def getVersionMajor(): - return versionMajor() -def getVersionMinor(): - return versionMinor() -def getVersionPatch(): - return versionPatch() -def getSha1(): - return sha1() -def getDebugInformation(): - return debugInformation() -def getPluginInstallPath(): - return pluginInstallPath() + +def getInfo(): + return SimpleNamespace( + version=versionString(), + major=versionMajor(), + minor=versionMinor(), + patch=versionPatch(), + debug=debugInformation(), + sha1=sha1(), + plugin=pluginInstallPath(), + ) + cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: From 04ee8ef17bc48d61f0648c12907be85c92e6a543 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 8 Sep 2021 14:07:50 +0300 Subject: [PATCH 022/161] Refactor Pipeline - Rename PyPipeline to Pipeline and drop pure Python Pipeline - Rename Pipeline to PyPipelineExecutor - Replace composition with inheritance between PyPipelineExecutor and PipelineExecutor - Refactor libpdalpython.pyx --- pdal/PyPipeline.cpp | 63 +++++-------------- pdal/PyPipeline.hpp | 30 +-------- pdal/__init__.py | 2 +- pdal/libpdalpython.pyx | 136 ++++++++++++++++++++--------------------- pdal/pipeline.py | 70 --------------------- 5 files changed, 83 insertions(+), 218 deletions(-) delete mode 100644 pdal/pipeline.py diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 215652f6..05bc0ab0 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -53,8 +53,8 @@ namespace python { // Create a pipeline for writing data to PDAL -Pipeline::Pipeline(std::string const& json, std::vector arrays) : - m_executor(new PipelineExecutor(json)) +PyPipelineExecutor::PyPipelineExecutor(std::string const& json, + std::vector arrays) : PipelineExecutor(json) { #ifndef _WIN32 // See comment in alternate constructor below. @@ -64,8 +64,7 @@ Pipeline::Pipeline(std::string const& json, std::vector arrays) : if (_import_array() < 0) throw pdal_error("Could not impory numpy.core.multiarray."); - PipelineManager& manager = m_executor->getManager(); - + PipelineManager& manager = getManager(); std::stringstream strm(json); manager.readPipeline(strm); std::vector roots = manager.roots(); @@ -107,8 +106,7 @@ Pipeline::Pipeline(std::string const& json, std::vector arrays) : } // Create a pipeline for reading data from PDAL -Pipeline::Pipeline(std::string const& json) : - m_executor(new PipelineExecutor(json)) +PyPipelineExecutor::PyPipelineExecutor(std::string const& json) : PipelineExecutor(json) { // Make the symbols in pdal_base global so that they're accessible // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, @@ -124,65 +122,32 @@ Pipeline::Pipeline(std::string const& json) : throw pdal_error("Could not impory numpy.core.multiarray."); } -Pipeline::~Pipeline() -{} - - -void Pipeline::setLogLevel(int level) -{ - m_executor->setLogLevel(level); -} - - -int Pipeline::getLogLevel() const +std::vector PyPipelineExecutor::getArrays() const { - return static_cast(m_executor->getLogLevel()); -} - - -int64_t Pipeline::execute() -{ - return m_executor->execute(); -} - -bool Pipeline::validate() -{ - auto res = m_executor->validate(); - return res; -} - -std::vector Pipeline::getArrays() const -{ - std::vector output; - - if (!m_executor->executed()) + if (!executed()) throw python_error("call execute() before fetching arrays"); - const PointViewSet& pvset = m_executor->getManagerConst().views(); - - for (auto i: pvset) + std::vector output; + for (auto view: getManagerConst().views()) { //ABELL - Leak? Array *array = new python::Array; - array->update(i); + array->update(view); output.push_back(array); } return output; } -std::vector Pipeline::getMeshes() const +std::vector PyPipelineExecutor::getMeshes() const { - std::vector output; - - if (!m_executor->executed()) + if (!executed()) throw python_error("call execute() before fetching the mesh"); - const PointViewSet& pvset = m_executor->getManagerConst().views(); - - for (auto i: pvset) + std::vector output; + for (auto view: getManagerConst().views()) { Mesh *mesh = new python::Mesh; - mesh->update(i); + mesh->update(view); output.push_back(mesh); } return output; diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index ef186d6a..9339e41b 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -58,39 +58,15 @@ class python_error : public std::runtime_error {} }; -class Pipeline +class PyPipelineExecutor : public PipelineExecutor { public: - Pipeline(std::string const& json); - Pipeline(std::string const& json, + PyPipelineExecutor(std::string const& json); + PyPipelineExecutor(std::string const& json, std::vector arrays); - ~Pipeline(); - int64_t execute(); - bool validate(); - inline std::string getPipeline() const - { - return m_executor->getPipeline(); - } - inline std::string getMetadata() const - { - return m_executor->getMetadata(); - } - inline std::string getSchema() const - { - return m_executor->getSchema(); - } - inline std::string getLog() const - { - return m_executor->getLog(); - } std::vector getArrays() const; std::vector getMeshes() const; - void setLogLevel(int level); - int getLogLevel() const; - -private: - std::shared_ptr m_executor; }; } // namespace python diff --git a/pdal/__init__.py b/pdal/__init__.py index 5109f32c..6ce05262 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,5 +1,5 @@ +from .libpdalpython import Pipeline from .libpdalpython import getDimensions, getInfo -from .pipeline import Pipeline __version__ = "2.4.2" diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index cafd6ae1..170a726b 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -1,16 +1,20 @@ # distutils: language = c++ # cython: c_string_type=unicode, c_string_encoding=utf8 +import json from libcpp.vector cimport vector from libcpp.string cimport string from libc.stdint cimport int64_t from libcpp cimport bool from types import SimpleNamespace + +import numpy as np cimport numpy as np np.import_array() from cython.operator cimport dereference as deref, preincrement as inc + cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": cdef int versionMajor() except + cdef int versionMinor() except + @@ -20,7 +24,6 @@ cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": cdef string pluginInstallPath() except+ cdef string versionString() except+ - def getInfo(): return SimpleNamespace( version=versionString(), @@ -33,6 +36,26 @@ def getInfo(): ) +cdef extern from "PyDimension.hpp": + ctypedef struct Dimension: + string name; + string description; + int size; + string type; +## string units; // Not defined by PDAL yet + cdef vector[Dimension] getValidDimensions() except + + +def getDimensions(): + output = [] + for dim in getValidDimensions(): + output.append({ + 'name': dim.name, + 'description': dim.description, + 'dtype': np.dtype(dim.type + str(dim.size)) + }) + return output + + cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: Array(np.ndarray) except + @@ -40,13 +63,12 @@ cdef extern from "PyArray.hpp" namespace "pdal::python": cdef extern from "PyMesh.hpp" namespace "pdal::python": cdef cppclass Mesh: - Mesh(np.ndarray) except + void *getPythonArray() except + cdef extern from "PyPipeline.hpp" namespace "pdal::python": - cdef cppclass Pipeline: - Pipeline(const char* ) except + - Pipeline(const char*, vector[Array*]& ) except + + cdef cppclass PyPipelineExecutor: + PyPipelineExecutor(const char*) except + + PyPipelineExecutor(const char*, vector[Array*]&) except + int64_t execute() except + bool validate() except + string getPipeline() except + @@ -58,118 +80,90 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": int getLogLevel() void setLogLevel(int) -cdef extern from "PyDimension.hpp": - ctypedef struct Dimension: - string name; - string description; - int size; - string type; -## string units; // Not defined by PDAL yet - - cdef vector[Dimension] getValidDimensions() except + - -def getDimensions(): - cdef vector[Dimension] c_dims; - c_dims = getValidDimensions() - output = [] - cdef vector[Dimension].iterator it = c_dims.begin() - while it != c_dims.end(): - ptr = deref(it) - d = {} - d['name'] = ptr.name - d['description'] = ptr.description - kind = ptr.type + str(ptr.size) - d['dtype'] = np.dtype(kind) - ptr = deref(it) - output.append(d) - inc(it) - return output - - -cdef class PyPipeline: - cdef Pipeline *thisptr # hold a c++ instance which we're wrapping - cdef vector[Array *] c_arrays; +cdef class Pipeline: + cdef PyPipelineExecutor* _executor + cdef vector[Array *] _arrays; def __cinit__(self, unicode json, list arrays=None): - cdef char* x = NULL - if arrays is not None: for array in arrays: - self.c_arrays.push_back(new Array(array)) - self.thisptr = new Pipeline(json.encode('UTF-8'), self.c_arrays) + self._arrays.push_back(new Array(array)) + self._executor = new PyPipelineExecutor(json.encode('UTF-8'), self._arrays) else: - self.thisptr = new Pipeline(json.encode('UTF-8')) + self._executor = new PyPipelineExecutor(json.encode('UTF-8')) def __dealloc__(self): - for array in self.c_arrays: + for array in self._arrays: del array - del self.thisptr + del self._executor property pipeline: def __get__(self): - return self.thisptr.getPipeline() + return self._executor.getPipeline() property metadata: def __get__(self): - return self.thisptr.getMetadata() + return self._executor.getMetadata() property loglevel: def __get__(self): - return self.thisptr.getLogLevel() + return self._executor.getLogLevel() def __set__(self, v): - self.thisptr.setLogLevel(v) + self._executor.setLogLevel(v) property log: def __get__(self): - - return self.thisptr.getLog() + return self._executor.getLog() property schema: def __get__(self): - import json - - j = self.thisptr.getSchema() - return json.loads(j) + return json.loads(self._executor.getSchema()) property arrays: - def __get__(self): - v = self.thisptr.getArrays() output = [] + v = self._executor.getArrays() cdef vector[Array*].iterator it = v.begin() - cdef Array* a + cdef Array* ptr while it != v.end(): ptr = deref(it) - a = ptr#.get() - o = a.getPythonArray() - output.append(o) + output.append(ptr.getPythonArray()) del ptr inc(it) return output property meshes: - def __get__(self): - v = self.thisptr.getMeshes() output = [] + v = self._executor.getMeshes() cdef vector[Mesh *].iterator it = v.begin() - cdef Mesh* m + cdef Mesh* ptr while it != v.end(): ptr = deref(it) - m = ptr#.get() - o = m.getPythonArray() - output.append(o) + output.append(ptr.getPythonArray()) del ptr inc(it) return output def execute(self): - if not self.thisptr: - raise Exception("C++ Pipeline object not constructed!") - return self.thisptr.execute() + return self._executor.execute() def validate(self): - if not self.thisptr: - raise Exception("C++ Pipeline object not constructed!") - return self.thisptr.validate() + return self._executor.validate() + + def get_meshio(self, idx): + try: + from meshio import Mesh + except ModuleNotFoundError: + raise RuntimeError( + "The get_meshio function can only be used if you have installed meshio. Try pip install meshio" + ) + array = self.arrays[idx] + mesh = self.meshes[idx] + if len(mesh) == 0: + return None + return Mesh( + np.stack((array["X"], array["Y"], array["Z"]), 1), + [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], + ) diff --git a/pdal/pipeline.py b/pdal/pipeline.py deleted file mode 100644 index e9d502a0..00000000 --- a/pdal/pipeline.py +++ /dev/null @@ -1,70 +0,0 @@ - -from pdal import libpdalpython -import numpy as np - -class Pipeline(object): - """A PDAL pipeline object, defined by JSON. See http://www.pdal.io/pipeline.html for more - information on how to define one""" - - def __init__(self, json, arrays=None): - - if arrays: - self.p = libpdalpython.PyPipeline(json, arrays) - else: - self.p = libpdalpython.PyPipeline(json) - - def get_metadata(self): - return self.p.metadata - metadata = property(get_metadata) - - def get_schema(self): - return self.p.schema - schema = property(get_schema) - - def get_pipeline(self): - return self.p.pipeline - pipeline = property(get_pipeline) - - def get_loglevel(self): - return self.p.loglevel - - def set_loglevel(self, v): - self.p.loglevel = v - loglevel = property(get_loglevel, set_loglevel) - - def get_log(self): - return self.p.log - log = property(get_log) - - def execute(self): - return self.p.execute() - - def validate(self): - return self.p.validate() - - def get_arrays(self): - return self.p.arrays - arrays = property(get_arrays) - - def get_meshes(self): - return self.p.meshes - meshes = property(get_meshes) - - def get_meshio(self, idx: int): - try: - from meshio import Mesh - except ModuleNotFoundError: - raise RuntimeError( - "The get_meshio function can only be used if you have installed meshio. Try pip install meshio") - array = self.arrays[idx] - mesh = self.meshes[idx] - if len(mesh) == 0: - return None - return Mesh( - np.stack((array['X'], array['Y'], array['Z']), 1), - [( - "triangle", - np.stack((mesh['A'], mesh['B'], mesh['C']), 1) - ) - ] - ) \ No newline at end of file From 807ed64c0ea0de85987a53f7582d91f3cdcc0bf2 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 29 Sep 2021 09:20:23 +0300 Subject: [PATCH 023/161] Refactor Array/Mesh classes --- pdal/CMakeLists.txt | 2 - pdal/PyArray.cpp | 168 ++++++++++++++++++++------------ pdal/PyArray.hpp | 53 ++++++---- pdal/PyMesh.cpp | 216 ----------------------------------------- pdal/PyMesh.hpp | 103 -------------------- pdal/PyPipeline.cpp | 19 +--- pdal/libpdalpython.pyx | 2 +- 7 files changed, 143 insertions(+), 420 deletions(-) delete mode 100644 pdal/PyMesh.cpp delete mode 100644 pdal/PyMesh.hpp diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt index 74b2d4dd..97bc051a 100644 --- a/pdal/CMakeLists.txt +++ b/pdal/CMakeLists.txt @@ -1,8 +1,6 @@ set(EXTENSION_SRC PyArray.cpp PyArray.hpp - PyMesh.cpp - PyMesh.hpp PyDimension.hpp PyPipeline.cpp PyPipeline.hpp) diff --git a/pdal/PyArray.cpp b/pdal/PyArray.cpp index 73e6410d..28d0630a 100644 --- a/pdal/PyArray.cpp +++ b/pdal/PyArray.cpp @@ -34,7 +34,6 @@ #include "PyArray.hpp" #include - #include namespace pdal @@ -90,12 +89,36 @@ std::string toString(PyObject *pname) } // unnamed namespace -Array::Array() : m_array(nullptr) +Array::Array(PointViewPtr view) { if (_import_array() < 0) throw pdal_error("Could not import numpy.core.multiarray."); + + PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); + if (!dtype_dict) + throw pdal_error("Unable to build numpy dtype " + "description dictionary"); + + PyArray_Descr *dtype = nullptr; + if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) + throw pdal_error("Unable to build numpy dtype"); + Py_XDECREF(dtype_dict); + + // This is a 1 x size array. + npy_intp size = view->size(); + m_array = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); + + // copy the data + DimTypeList types = view->dimTypes(); + for (PointId idx = 0; idx < view->size(); idx++) + { + char *p = (char *)PyArray_GETPTR1(m_array, idx); + view->getPackedPoint(types, idx, p); + } } + Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) { if (_import_array() < 0) @@ -163,50 +186,10 @@ Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) Array::~Array() { - if (m_array) - Py_XDECREF((PyObject *)m_array); + Py_XDECREF(m_array); } -void Array::update(PointViewPtr view) -{ - if (m_array) - Py_XDECREF((PyObject *)m_array); - m_array = nullptr; // Just in case of an exception. - - Dimension::IdList dims = view->dims(); - npy_intp size = view->size(); - - PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); - if (!dtype_dict) - throw pdal_error("Unable to build numpy dtype " - "description dictionary"); - - PyArray_Descr *dtype = nullptr; - if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) - throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dtype_dict); - - // This is a 1 x size array. - m_array = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, - 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); - - // copy the data - DimTypeList types = view->dimTypes(); - for (PointId idx = 0; idx < view->size(); idx++) - { - char *p = (char *)PyArray_GETPTR1(m_array, idx); - view->getPackedPoint(types, idx, p); - } -} - - -//ABELL - Who's responsible for incrementing the ref count? -PyArrayObject *Array::getPythonArray() const -{ - return m_array; -} - PyObject* Array::buildNumpyDescription(PointViewPtr view) const { // Build up a numpy dtype dictionary @@ -263,31 +246,100 @@ PyObject* Array::buildNumpyDescription(PointViewPtr view) const return dict; } -bool Array::rowMajor() const + +ArrayIter& Array::iterator() { - return m_rowMajor; + ArrayIter *it = new ArrayIter(m_array); + m_iterators.push_back(std::unique_ptr(it)); + return *it; } -Array::Shape Array::shape() const + +Mesh::Mesh(PointViewPtr view) { - return m_shape; + if (_import_array() < 0) + throw pdal_error("Could not import numpy.core.multiarray."); + + PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); + if (!dtype_dict) + throw pdal_error("Unable to build numpy dtype " + "description dictionary"); + + PyArray_Descr *dtype = nullptr; + if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) + throw pdal_error("Unable to build numpy dtype"); + Py_XDECREF(dtype_dict); + + // This is a 1 x size array. + TriangularMesh* mesh = view->mesh(); + npy_intp size = mesh ? mesh->size() : 0; + m_mesh = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); + + for (PointId idx = 0; idx < size; idx++) + { + char* p = (char *)PyArray_GETPTR1(m_mesh, idx); + const Triangle& t = (*mesh)[idx]; + uint32_t a = (uint32_t)t.m_a; + std::memcpy(p, &a, 4); + uint32_t b = (uint32_t)t.m_b; + std::memcpy(p + 4, &b, 4); + uint32_t c = (uint32_t)t.m_c; + std::memcpy(p + 8, &c, 4); + } } -const Array::Fields& Array::fields() const + +Mesh::~Mesh() { - return m_fields; + if (m_mesh) + Py_XDECREF((PyObject *)m_mesh); } -ArrayIter& Array::iterator() + +PyObject* Mesh::buildNumpyDescription(PointViewPtr view) const +{ + // Build up a numpy dtype dictionary + // + // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', + // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], + // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', + // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', + // 'Classification', 'ScanAngleRank', 'UserData', + // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} + // + + Dimension::IdList dims = view->dims(); + + PyObject* dict = PyDict_New(); + PyObject* formats = PyList_New(3); + PyObject* titles = PyList_New(3); + + PyList_SetItem(titles, 0, PyUnicode_FromString("A")); + PyList_SetItem(formats, 0, PyUnicode_FromString("u4")); + PyList_SetItem(titles, 1, PyUnicode_FromString("B")); + PyList_SetItem(formats, 1, PyUnicode_FromString("u4")); + PyList_SetItem(titles, 2, PyUnicode_FromString("C")); + PyList_SetItem(formats, 2, PyUnicode_FromString("u4")); + + + PyDict_SetItemString(dict, "names", titles); + PyDict_SetItemString(dict, "formats", formats); + + return dict; +} + +ArrayIter& Mesh::iterator() { - ArrayIter *it = new ArrayIter(*this); + ArrayIter *it = new ArrayIter(m_mesh); m_iterators.push_back(std::unique_ptr(it)); return *it; } -ArrayIter::ArrayIter(Array& array) + +ArrayIter::ArrayIter(PyArrayObject* np_array) { - m_iter = NpyIter_New(array.getPythonArray(), + m_iter = NpyIter_New(np_array, NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, NPY_KEEPORDER, NPY_NO_CASTING, NULL); if (!m_iter) @@ -324,16 +376,6 @@ ArrayIter& ArrayIter::operator++() return *this; } -ArrayIter::operator bool () const -{ - return !m_done; -} - -char * ArrayIter::operator * () const -{ - return *m_data; -} - } // namespace python } // namespace pdal diff --git a/pdal/PyArray.hpp b/pdal/PyArray.hpp index 0117861c..1d4f6d2a 100644 --- a/pdal/PyArray.hpp +++ b/pdal/PyArray.hpp @@ -34,12 +34,9 @@ #pragma once -#include - #include #include - -#include +#include namespace pdal { @@ -48,6 +45,30 @@ namespace python class ArrayIter; + +class PDAL_DLL Mesh +{ +public: + using Shape = std::array; + + Mesh(PointViewPtr view); + ~Mesh(); + + PyArrayObject *getPythonArray() const { return m_mesh; } + bool rowMajor() const { return m_rowMajor; } + Shape shape() const { return m_shape; } + ArrayIter& iterator(); + +private: + inline PyObject* buildNumpyDescription(PointViewPtr view) const; + + PyArrayObject* m_mesh; + bool m_rowMajor; + Shape m_shape {}; + std::vector> m_iterators; +}; + + class PDAL_DLL Array { public: @@ -55,44 +76,40 @@ class PDAL_DLL Array using Fields = std::vector; // Create an array for reading data from PDAL. - Array(); - + Array(PointViewPtr view); // Create an array for writing data to PDAL. Array(PyArrayObject* array); - ~Array(); - void update(PointViewPtr view); - PyArrayObject *getPythonArray() const; - bool rowMajor() const; - Shape shape() const; - const Fields& fields() const; - ArrayIter& iterator(); + PyArrayObject *getPythonArray() const { return m_array; } + bool rowMajor() const { return m_rowMajor; }; + Shape shape() const { return m_shape; } + const Fields& fields() const { return m_fields; }; + ArrayIter& iterator(); private: inline PyObject* buildNumpyDescription(PointViewPtr view) const; - PyArrayObject* m_array; - Array& operator=(Array const& rhs); Fields m_fields; bool m_rowMajor; Shape m_shape {}; std::vector> m_iterators; }; + class ArrayIter { public: ArrayIter(const ArrayIter&) = delete; ArrayIter() = delete; - ArrayIter(Array& array); + ArrayIter(PyArrayObject*); ~ArrayIter(); ArrayIter& operator++(); - operator bool () const; - char *operator * () const; + operator bool () const { return !m_done; } + char* operator*() const { return *m_data; } private: NpyIter *m_iter; diff --git a/pdal/PyMesh.cpp b/pdal/PyMesh.cpp deleted file mode 100644 index 729e6a5c..00000000 --- a/pdal/PyMesh.cpp +++ /dev/null @@ -1,216 +0,0 @@ -/****************************************************************************** -* Copyright (c) 2021, Runette Software Ltd (www.runette.co.uk) -* -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following -* conditions are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided -* with the distribution. -* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the -* names of its contributors may be used to endorse or promote -* products derived from this software without specific prior -* written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -* OF SUCH DAMAGE. -****************************************************************************/ - -#include "PyMesh.hpp" - -#include - - -namespace pdal -{ -namespace python -{ - -// Create new empty mesh -// -Mesh::Mesh() : m_mesh(nullptr) -{ - hasMesh = false; - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); -} - -Mesh::~Mesh() -{ - if (m_mesh) - Py_XDECREF((PyObject *)m_mesh); -} - -// Update from a PointView -// -void Mesh::update(PointViewPtr view) -{ - npy_intp size; - - if (m_mesh) - Py_XDECREF((PyObject *)m_mesh); - m_mesh = nullptr; // Just in case of an exception. - - TriangularMesh* mesh = view->mesh(); - if (mesh) - { - hasMesh = true; - size = mesh->size(); - } else { - hasMesh = false; - size = 0; - } - - PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); - if (!dtype_dict) - throw pdal_error("Unable to build numpy dtype " - "description dictionary"); - - PyArray_Descr *dtype = nullptr; - if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) - throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dtype_dict); - - // This is a 1 x size array. - m_mesh = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, - 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); - - for (PointId idx = 0; idx < size; idx++) - { - char* p = (char *)PyArray_GETPTR1(m_mesh, idx); - const Triangle& t = (*mesh)[idx]; - uint32_t a = (uint32_t)t.m_a; - std::memcpy(p, &a, 4); - uint32_t b = (uint32_t)t.m_b; - std::memcpy(p + 4, &b, 4); - uint32_t c = (uint32_t)t.m_c; - std::memcpy(p + 8, &c, 4); - } -} - - - -PyArrayObject *Mesh::getPythonArray() const -{ - return m_mesh; -} - -PyObject* Mesh::buildNumpyDescription(PointViewPtr view) const -{ - // Build up a numpy dtype dictionary - // - // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', - // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], - // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', - // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', - // 'Classification', 'ScanAngleRank', 'UserData', - // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} - // - - Dimension::IdList dims = view->dims(); - - PyObject* dict = PyDict_New(); - PyObject* formats = PyList_New(3); - PyObject* titles = PyList_New(3); - - PyList_SetItem(titles, 0, PyUnicode_FromString("A")); - PyList_SetItem(formats, 0, PyUnicode_FromString("u4")); - PyList_SetItem(titles, 1, PyUnicode_FromString("B")); - PyList_SetItem(formats, 1, PyUnicode_FromString("u4")); - PyList_SetItem(titles, 2, PyUnicode_FromString("C")); - PyList_SetItem(formats, 2, PyUnicode_FromString("u4")); - - - PyDict_SetItemString(dict, "names", titles); - PyDict_SetItemString(dict, "formats", formats); - - return dict; -} - -bool Mesh::rowMajor() const -{ - return m_rowMajor; -} - -Mesh::Shape Mesh::shape() const -{ - return m_shape; -} - - -MeshIter& Mesh::iterator() -{ - MeshIter *it = new MeshIter(*this); - m_iterators.push_back(std::unique_ptr(it)); - return *it; -} - -MeshIter::MeshIter(Mesh& mesh) -{ - m_iter = NpyIter_New(mesh.getPythonArray(), - NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, - NPY_KEEPORDER, NPY_NO_CASTING, NULL); - if (!m_iter) - throw pdal_error("Unable to create numpy iterator."); - - char *itererr; - m_iterNext = NpyIter_GetIterNext(m_iter, &itererr); - if (!m_iterNext) - { - NpyIter_Deallocate(m_iter); - throw pdal_error(std::string("Unable to create numpy iterator: ") + - itererr); - } - m_data = NpyIter_GetDataPtrArray(m_iter); - m_stride = NpyIter_GetInnerStrideArray(m_iter); - m_size = NpyIter_GetInnerLoopSizePtr(m_iter); - m_done = false; -} - -MeshIter::~MeshIter() -{ - NpyIter_Deallocate(m_iter); -} - -MeshIter& MeshIter::operator++() -{ - if (m_done) - return *this; - - if (--(*m_size)) - *m_data += *m_stride; - else if (!m_iterNext(m_iter)) - m_done = true; - return *this; -} - -MeshIter::operator bool () const -{ - return !m_done; -} - -char * MeshIter::operator * () const -{ - return *m_data; -} - - -} // namespace python -} // namespace pdal - diff --git a/pdal/PyMesh.hpp b/pdal/PyMesh.hpp deleted file mode 100644 index 6b8b027d..00000000 --- a/pdal/PyMesh.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/****************************************************************************** -* Copyright (c) 2021, Runette Software Ltd (www.runette.co.uk) -* -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following -* conditions are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided -* with the distribution. -* * Neither the name of Hobu, Inc. nor the -* names of its contributors may be used to endorse or promote -* products derived from this software without specific prior -* written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -* OF SUCH DAMAGE. -****************************************************************************/ - -#pragma once - - -#include -#include - -#include - -namespace pdal -{ -namespace python -{ -class MeshIter; - -class PDAL_DLL Mesh -{ -public: - using Shape = std::array; - - bool hasMesh; - - Mesh(); - - ~Mesh(); - - void update(PointViewPtr view); - PyArrayObject *getPythonArray() const; - - bool rowMajor() const; - Shape shape() const; - MeshIter& iterator(); - - -private: - inline PyObject* buildNumpyDescription(PointViewPtr view) const; - - PyArrayObject* m_mesh; - - Mesh& operator=(Mesh const& rhs); - bool m_rowMajor; - Shape m_shape {}; - std::vector> m_iterators; -}; - -class MeshIter -{ -public: - MeshIter(const MeshIter&) = delete; - MeshIter() = delete; - - MeshIter(Mesh& mesh); - ~MeshIter(); - - MeshIter& operator++(); - operator bool () const; - char *operator * () const; - -private: - NpyIter *m_iter; - NpyIter_IterNextFunc *m_iterNext; - char **m_data; - npy_intp *m_size; - npy_intp *m_stride; - bool m_done; -}; - -} // namespace python -} // namespace pdal - diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 05bc0ab0..5a4a78e9 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -45,7 +45,6 @@ #include #include "PyArray.hpp" -#include "PyMesh.hpp" namespace pdal { @@ -61,9 +60,6 @@ PyPipelineExecutor::PyPipelineExecutor(std::string const& json, ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); #endif - if (_import_array() < 0) - throw pdal_error("Could not impory numpy.core.multiarray."); - PipelineManager& manager = getManager(); std::stringstream strm(json); manager.readPipeline(strm); @@ -118,8 +114,6 @@ PyPipelineExecutor::PyPipelineExecutor(std::string const& json) : PipelineExecut #ifndef _WIN32 ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); #endif - if (_import_array() < 0) - throw pdal_error("Could not impory numpy.core.multiarray."); } std::vector PyPipelineExecutor::getArrays() const @@ -129,12 +123,8 @@ std::vector PyPipelineExecutor::getArrays() const std::vector output; for (auto view: getManagerConst().views()) - { //ABELL - Leak? - Array *array = new python::Array; - array->update(view); - output.push_back(array); - } + output.push_back(new python::Array(view)); return output; } @@ -145,14 +135,9 @@ std::vector PyPipelineExecutor::getMeshes() const std::vector output; for (auto view: getManagerConst().views()) - { - Mesh *mesh = new python::Mesh; - mesh->update(view); - output.push_back(mesh); - } + output.push_back(new python::Mesh(view)); return output; } } // namespace python } // namespace pdal - diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 170a726b..33240d1b 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -61,10 +61,10 @@ cdef extern from "PyArray.hpp" namespace "pdal::python": Array(np.ndarray) except + void *getPythonArray() except+ -cdef extern from "PyMesh.hpp" namespace "pdal::python": cdef cppclass Mesh: void *getPythonArray() except + + cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef cppclass PyPipelineExecutor: PyPipelineExecutor(const char*) except + From 1c77b7a1dbace01035edbd2f5d151bbe0f28c0ef Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 29 Sep 2021 13:56:50 +0300 Subject: [PATCH 024/161] Replace Mesh class with meshToNumpyArray function --- pdal/PyArray.cpp | 168 ----------------------------------------- pdal/PyArray.hpp | 29 ------- pdal/PyPipeline.cpp | 134 ++++++++++++++++++++++++++++++-- pdal/PyPipeline.hpp | 12 +-- pdal/libpdalpython.pyx | 32 +------- 5 files changed, 133 insertions(+), 242 deletions(-) diff --git a/pdal/PyArray.cpp b/pdal/PyArray.cpp index 28d0630a..db219f01 100644 --- a/pdal/PyArray.cpp +++ b/pdal/PyArray.cpp @@ -89,35 +89,6 @@ std::string toString(PyObject *pname) } // unnamed namespace -Array::Array(PointViewPtr view) -{ - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); - - PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); - if (!dtype_dict) - throw pdal_error("Unable to build numpy dtype " - "description dictionary"); - - PyArray_Descr *dtype = nullptr; - if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) - throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dtype_dict); - - // This is a 1 x size array. - npy_intp size = view->size(); - m_array = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, - 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); - - // copy the data - DimTypeList types = view->dimTypes(); - for (PointId idx = 0; idx < view->size(); idx++) - { - char *p = (char *)PyArray_GETPTR1(m_array, idx); - view->getPackedPoint(types, idx, p); - } -} - Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) { @@ -190,63 +161,6 @@ Array::~Array() } -PyObject* Array::buildNumpyDescription(PointViewPtr view) const -{ - // Build up a numpy dtype dictionary - // - // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', - // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], - // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', - // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', - // 'Classification', 'ScanAngleRank', 'UserData', - // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} - // - - Dimension::IdList dims = view->dims(); - - PyObject* dict = PyDict_New(); - PyObject* sizes = PyList_New(dims.size()); - PyObject* formats = PyList_New(dims.size()); - PyObject* titles = PyList_New(dims.size()); - - for (size_t i = 0; i < dims.size(); ++i) - { - Dimension::Id id = dims[i]; - Dimension::Type t = view->dimType(id); - npy_intp stride = view->dimSize(id); - - std::string name = view->dimName(id); - - std::string kind("i"); - Dimension::BaseType b = Dimension::base(t); - if (b == Dimension::BaseType::Unsigned) - kind = "u"; - else if (b == Dimension::BaseType::Signed) - kind = "i"; - else if (b == Dimension::BaseType::Floating) - kind = "f"; - else - throw pdal_error("Unable to map kind '" + kind + - "' to PDAL dimension type"); - - std::stringstream oss; - oss << kind << stride; - PyObject* pySize = PyLong_FromLong(stride); - PyObject* pyTitle = PyUnicode_FromString(name.c_str()); - PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str()); - - PyList_SetItem(sizes, i, pySize); - PyList_SetItem(titles, i, pyTitle); - PyList_SetItem(formats, i, pyFormat); - } - - PyDict_SetItemString(dict, "names", titles); - PyDict_SetItemString(dict, "formats", formats); - - return dict; -} - - ArrayIter& Array::iterator() { ArrayIter *it = new ArrayIter(m_array); @@ -255,88 +169,6 @@ ArrayIter& Array::iterator() } -Mesh::Mesh(PointViewPtr view) -{ - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); - - PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); - if (!dtype_dict) - throw pdal_error("Unable to build numpy dtype " - "description dictionary"); - - PyArray_Descr *dtype = nullptr; - if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) - throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dtype_dict); - - // This is a 1 x size array. - TriangularMesh* mesh = view->mesh(); - npy_intp size = mesh ? mesh->size() : 0; - m_mesh = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, - 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); - - for (PointId idx = 0; idx < size; idx++) - { - char* p = (char *)PyArray_GETPTR1(m_mesh, idx); - const Triangle& t = (*mesh)[idx]; - uint32_t a = (uint32_t)t.m_a; - std::memcpy(p, &a, 4); - uint32_t b = (uint32_t)t.m_b; - std::memcpy(p + 4, &b, 4); - uint32_t c = (uint32_t)t.m_c; - std::memcpy(p + 8, &c, 4); - } -} - - -Mesh::~Mesh() -{ - if (m_mesh) - Py_XDECREF((PyObject *)m_mesh); -} - - -PyObject* Mesh::buildNumpyDescription(PointViewPtr view) const -{ - // Build up a numpy dtype dictionary - // - // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', - // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], - // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', - // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', - // 'Classification', 'ScanAngleRank', 'UserData', - // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} - // - - Dimension::IdList dims = view->dims(); - - PyObject* dict = PyDict_New(); - PyObject* formats = PyList_New(3); - PyObject* titles = PyList_New(3); - - PyList_SetItem(titles, 0, PyUnicode_FromString("A")); - PyList_SetItem(formats, 0, PyUnicode_FromString("u4")); - PyList_SetItem(titles, 1, PyUnicode_FromString("B")); - PyList_SetItem(formats, 1, PyUnicode_FromString("u4")); - PyList_SetItem(titles, 2, PyUnicode_FromString("C")); - PyList_SetItem(formats, 2, PyUnicode_FromString("u4")); - - - PyDict_SetItemString(dict, "names", titles); - PyDict_SetItemString(dict, "formats", formats); - - return dict; -} - -ArrayIter& Mesh::iterator() -{ - ArrayIter *it = new ArrayIter(m_mesh); - m_iterators.push_back(std::unique_ptr(it)); - return *it; -} - - ArrayIter::ArrayIter(PyArrayObject* np_array) { m_iter = NpyIter_New(np_array, diff --git a/pdal/PyArray.hpp b/pdal/PyArray.hpp index 1d4f6d2a..91d3e535 100644 --- a/pdal/PyArray.hpp +++ b/pdal/PyArray.hpp @@ -46,50 +46,21 @@ namespace python class ArrayIter; -class PDAL_DLL Mesh -{ -public: - using Shape = std::array; - - Mesh(PointViewPtr view); - ~Mesh(); - - PyArrayObject *getPythonArray() const { return m_mesh; } - bool rowMajor() const { return m_rowMajor; } - Shape shape() const { return m_shape; } - ArrayIter& iterator(); - -private: - inline PyObject* buildNumpyDescription(PointViewPtr view) const; - - PyArrayObject* m_mesh; - bool m_rowMajor; - Shape m_shape {}; - std::vector> m_iterators; -}; - - class PDAL_DLL Array { public: using Shape = std::array; using Fields = std::vector; - // Create an array for reading data from PDAL. - Array(PointViewPtr view); - // Create an array for writing data to PDAL. Array(PyArrayObject* array); ~Array(); - PyArrayObject *getPythonArray() const { return m_array; } bool rowMajor() const { return m_rowMajor; }; Shape shape() const { return m_shape; } const Fields& fields() const { return m_fields; }; ArrayIter& iterator(); private: - inline PyObject* buildNumpyDescription(PointViewPtr view) const; - PyArrayObject* m_array; Fields m_fields; bool m_rowMajor; diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 5a4a78e9..d1ff55f9 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -39,7 +39,6 @@ #endif #include -#include #include #include @@ -116,26 +115,145 @@ PyPipelineExecutor::PyPipelineExecutor(std::string const& json) : PipelineExecut #endif } -std::vector PyPipelineExecutor::getArrays() const +inline PyObject* buildNumpyDescription(PointViewPtr view) +{ + // Build up a numpy dtype dictionary + // + // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', + // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], + // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', + // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', + // 'Classification', 'ScanAngleRank', 'UserData', + // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} + // + Dimension::IdList dims = view->dims(); + PyObject* names = PyList_New(dims.size()); + PyObject* formats = PyList_New(dims.size()); + for (size_t i = 0; i < dims.size(); ++i) + { + Dimension::Id id = dims[i]; + std::string name = view->dimName(id); + npy_intp stride = view->dimSize(id); + + std::string kind; + Dimension::BaseType b = Dimension::base(view->dimType(id)); + if (b == Dimension::BaseType::Unsigned) + kind = "u"; + else if (b == Dimension::BaseType::Signed) + kind = "i"; + else if (b == Dimension::BaseType::Floating) + kind = "f"; + else + throw pdal_error("Unable to map kind '" + kind + + "' to PDAL dimension type"); + + std::stringstream oss; + oss << kind << stride; + PyList_SetItem(names, i, PyUnicode_FromString(name.c_str())); + PyList_SetItem(formats, i, PyUnicode_FromString(oss.str().c_str())); + } + PyObject* dtype_dict = PyDict_New(); + PyDict_SetItemString(dtype_dict, "names", names); + PyDict_SetItemString(dtype_dict, "formats", formats); + return dtype_dict; +} + + +PyArrayObject* viewToNumpyArray(PointViewPtr view) +{ + if (_import_array() < 0) + throw pdal_error("Could not import numpy.core.multiarray."); + + PyObject* dtype_dict = buildNumpyDescription(view); + PyArray_Descr *dtype = nullptr; + if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) + throw pdal_error("Unable to build numpy dtype"); + Py_XDECREF(dtype_dict); + + // This is a 1 x size array. + npy_intp size = view->size(); + PyArrayObject* array = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); + + // copy the data + DimTypeList types = view->dimTypes(); + for (PointId idx = 0; idx < view->size(); idx++) + view->getPackedPoint(types, idx, (char *)PyArray_GETPTR1(array, idx)); + return array; +} + +std::vector PyPipelineExecutor::getArrays() const { if (!executed()) throw python_error("call execute() before fetching arrays"); - std::vector output; + std::vector output; for (auto view: getManagerConst().views()) - //ABELL - Leak? - output.push_back(new python::Array(view)); + output.push_back(viewToNumpyArray(view)); return output; } -std::vector PyPipelineExecutor::getMeshes() const + +PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) +{ + if (_import_array() < 0) + throw pdal_error("Could not import numpy.core.multiarray."); + + // Build up a numpy dtype dictionary + // + // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', + // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], + // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', + // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', + // 'Classification', 'ScanAngleRank', 'UserData', + // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} + // + PyObject* names = PyList_New(3); + PyList_SetItem(names, 0, PyUnicode_FromString("A")); + PyList_SetItem(names, 1, PyUnicode_FromString("B")); + PyList_SetItem(names, 2, PyUnicode_FromString("C")); + + PyObject* formats = PyList_New(3); + PyList_SetItem(formats, 0, PyUnicode_FromString("u4")); + PyList_SetItem(formats, 1, PyUnicode_FromString("u4")); + PyList_SetItem(formats, 2, PyUnicode_FromString("u4")); + + PyObject* dtype_dict = PyDict_New(); + PyDict_SetItemString(dtype_dict, "names", names); + PyDict_SetItemString(dtype_dict, "formats", formats); + + PyArray_Descr *dtype = nullptr; + if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) + throw pdal_error("Unable to build numpy dtype"); + Py_XDECREF(dtype_dict); + + // This is a 1 x size array. + npy_intp size = mesh ? mesh->size() : 0; + PyArrayObject* array = (PyArrayObject*)PyArray_NewFromDescr(&PyArray_Type, dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); + for (PointId idx = 0; idx < size; idx++) + { + char* p = (char *)PyArray_GETPTR1(array, idx); + const Triangle& t = (*mesh)[idx]; + uint32_t a = (uint32_t)t.m_a; + std::memcpy(p, &a, 4); + uint32_t b = (uint32_t)t.m_b; + std::memcpy(p + 4, &b, 4); + uint32_t c = (uint32_t)t.m_c; + std::memcpy(p + 8, &c, 4); + } + return array; +} + + +std::vector PyPipelineExecutor::getMeshes() const { if (!executed()) throw python_error("call execute() before fetching the mesh"); - std::vector output; + std::vector output; for (auto view: getManagerConst().views()) - output.push_back(new python::Mesh(view)); + output.push_back(meshToNumpyArray(view->mesh())); return output; } diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index 9339e41b..266bd9fd 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -34,14 +34,9 @@ #pragma once -#include -#include -#include #include -#include -#include -#include +#include namespace pdal { @@ -49,7 +44,6 @@ namespace python { class Array; -class Mesh; class python_error : public std::runtime_error { @@ -65,8 +59,8 @@ class PyPipelineExecutor : public PipelineExecutor PyPipelineExecutor(std::string const& json, std::vector arrays); - std::vector getArrays() const; - std::vector getMeshes() const; + std::vector getArrays() const; + std::vector getMeshes() const; }; } // namespace python diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 33240d1b..6d8c3d65 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -12,8 +12,6 @@ import numpy as np cimport numpy as np np.import_array() -from cython.operator cimport dereference as deref, preincrement as inc - cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": cdef int versionMajor() except + @@ -59,10 +57,6 @@ def getDimensions(): cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: Array(np.ndarray) except + - void *getPythonArray() except+ - - cdef cppclass Mesh: - void *getPythonArray() except + cdef extern from "PyPipeline.hpp" namespace "pdal::python": @@ -75,8 +69,8 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": string getMetadata() except + string getSchema() except + string getLog() except + - vector[Array*] getArrays() except + - vector[Mesh*] getMeshes() except + + vector[np.PyArrayObject*] getArrays() except + + vector[np.PyArrayObject*] getMeshes() except + int getLogLevel() void setLogLevel(int) @@ -122,29 +116,11 @@ cdef class Pipeline: property arrays: def __get__(self): - output = [] - v = self._executor.getArrays() - cdef vector[Array*].iterator it = v.begin() - cdef Array* ptr - while it != v.end(): - ptr = deref(it) - output.append(ptr.getPythonArray()) - del ptr - inc(it) - return output + return [a for a in self._executor.getArrays()] property meshes: def __get__(self): - output = [] - v = self._executor.getMeshes() - cdef vector[Mesh *].iterator it = v.begin() - cdef Mesh* ptr - while it != v.end(): - ptr = deref(it) - output.append(ptr.getPythonArray()) - del ptr - inc(it) - return output + return [a for a in self._executor.getMeshes()] def execute(self): return self._executor.execute() From a69b24828fbddb1a3823dc24102c0cb36d5e8c6d Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 29 Sep 2021 18:03:28 +0300 Subject: [PATCH 025/161] Fix memory leak in Pipeline.{arrays,meshes} properties --- pdal/libpdalpython.pyx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 6d8c3d65..2b76b73b 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -2,6 +2,7 @@ # cython: c_string_type=unicode, c_string_encoding=utf8 import json +from cpython.ref cimport Py_DECREF from libcpp.vector cimport vector from libcpp.string cimport string from libc.stdint cimport int64_t @@ -116,11 +117,11 @@ cdef class Pipeline: property arrays: def __get__(self): - return [a for a in self._executor.getArrays()] + return self._vector_to_list(self._executor.getArrays()) property meshes: def __get__(self): - return [a for a in self._executor.getMeshes()] + return self._vector_to_list(self._executor.getMeshes()) def execute(self): return self._executor.execute() @@ -143,3 +144,10 @@ cdef class Pipeline: np.stack((array["X"], array["Y"], array["Z"]), 1), [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], ) + + cdef _vector_to_list(self, vector[np.PyArrayObject*] arrays): + output = [] + for array in arrays: + output.append(array) + Py_DECREF(output[-1]) + return output From 332186ecd893bd5e0c99235815409ec28d0eccf3 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 30 Sep 2021 16:31:12 +0300 Subject: [PATCH 026/161] Raise RuntimeError from Cython if the pipeline has not been executed --- pdal/PyPipeline.cpp | 6 ------ pdal/PyPipeline.hpp | 11 +---------- pdal/libpdalpython.pyx | 5 +++++ 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index d1ff55f9..33e1deb2 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -184,9 +184,6 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view) std::vector PyPipelineExecutor::getArrays() const { - if (!executed()) - throw python_error("call execute() before fetching arrays"); - std::vector output; for (auto view: getManagerConst().views()) output.push_back(viewToNumpyArray(view)); @@ -248,9 +245,6 @@ PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) std::vector PyPipelineExecutor::getMeshes() const { - if (!executed()) - throw python_error("call execute() before fetching the mesh"); - std::vector output; for (auto view: getManagerConst().views()) output.push_back(meshToNumpyArray(view->mesh())); diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index 266bd9fd..7713eab9 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -45,20 +45,11 @@ namespace python class Array; -class python_error : public std::runtime_error -{ -public: - inline python_error(std::string const& msg) : std::runtime_error(msg) - {} -}; - class PyPipelineExecutor : public PipelineExecutor { public: PyPipelineExecutor(std::string const& json); - PyPipelineExecutor(std::string const& json, - std::vector arrays); - + PyPipelineExecutor(std::string const& json, std::vector arrays); std::vector getArrays() const; std::vector getMeshes() const; }; diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 2b76b73b..e4fb31d6 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -64,6 +64,7 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef cppclass PyPipelineExecutor: PyPipelineExecutor(const char*) except + PyPipelineExecutor(const char*, vector[Array*]&) except + + bool executed() except + int64_t execute() except + bool validate() except + string getPipeline() except + @@ -117,10 +118,14 @@ cdef class Pipeline: property arrays: def __get__(self): + if not self._executor.executed(): + raise RuntimeError("call execute() before fetching arrays") return self._vector_to_list(self._executor.getArrays()) property meshes: def __get__(self): + if not self._executor.executed(): + raise RuntimeError("call execute() before fetching the mesh") return self._vector_to_list(self._executor.getMeshes()) def execute(self): From 09d994cbb51e561453908e104790ca2727f46ba8 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 30 Sep 2021 16:49:42 +0300 Subject: [PATCH 027/161] Drop PyPipelineExecutor class, convert its methods to plain functions --- pdal/PyPipeline.cpp | 49 +++++++++++++++++++----------------------- pdal/PyPipeline.hpp | 14 ++++++------ pdal/libpdalpython.pyx | 31 +++++++++++++++----------- 3 files changed, 46 insertions(+), 48 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 33e1deb2..0b488ccb 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -32,6 +32,7 @@ * OF SUCH DAMAGE. ****************************************************************************/ +#include "PyArray.hpp" #include "PyPipeline.hpp" #ifndef _WIN32 @@ -43,25 +44,33 @@ #include #include -#include "PyArray.hpp" - namespace pdal { namespace python { -// Create a pipeline for writing data to PDAL -PyPipelineExecutor::PyPipelineExecutor(std::string const& json, - std::vector arrays) : PipelineExecutor(json) + +void readPipeline(PipelineExecutor* executor, std::string json) { + std::stringstream strm(json); + executor->getManager().readPipeline(strm); +} + + +void addArrayReaders(PipelineExecutor* executor, std::vector arrays) +{ + // Make the symbols in pdal_base global so that they're accessible + // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, + // which means that without this, symbols in libpdal_base aren't available + // for resolution of symbols on future runtime linking. This is an issue + // on Alpine and other Linux variants that don't use UNIQUE symbols + // for C++ template statics only. Without this, you end up with multiple + // copies of template statics. #ifndef _WIN32 - // See comment in alternate constructor below. ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); #endif - PipelineManager& manager = getManager(); - std::stringstream strm(json); - manager.readPipeline(strm); + PipelineManager& manager = executor->getManager(); std::vector roots = manager.roots(); if (roots.size() != 1) throw pdal_error("Filter pipeline must contain a single root stage."); @@ -100,20 +109,6 @@ PyPipelineExecutor::PyPipelineExecutor(std::string const& json, manager.validateStageOptions(); } -// Create a pipeline for reading data from PDAL -PyPipelineExecutor::PyPipelineExecutor(std::string const& json) : PipelineExecutor(json) -{ - // Make the symbols in pdal_base global so that they're accessible - // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, - // which means that without this, symbols in libpdal_base aren't available - // for resolution of symbols on future runtime linking. This is an issue - // on Alpine and other Linux variants that don't use UNIQUE symbols - // for C++ template statics only. Without this, you end up with multiple - // copies of template statics. -#ifndef _WIN32 - ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); -#endif -} inline PyObject* buildNumpyDescription(PointViewPtr view) { @@ -182,10 +177,10 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view) return array; } -std::vector PyPipelineExecutor::getArrays() const +std::vector getArrays(const PipelineExecutor* executor) { std::vector output; - for (auto view: getManagerConst().views()) + for (auto view: executor->getManagerConst().views()) output.push_back(viewToNumpyArray(view)); return output; } @@ -243,10 +238,10 @@ PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) } -std::vector PyPipelineExecutor::getMeshes() const +std::vector getMeshes(const PipelineExecutor* executor) { std::vector output; - for (auto view: getManagerConst().views()) + for (auto view: executor->getManagerConst().views()) output.push_back(meshToNumpyArray(view->mesh())); return output; } diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index 7713eab9..b6d9c3f4 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -45,14 +45,12 @@ namespace python class Array; -class PyPipelineExecutor : public PipelineExecutor -{ -public: - PyPipelineExecutor(std::string const& json); - PyPipelineExecutor(std::string const& json, std::vector arrays); - std::vector getArrays() const; - std::vector getMeshes() const; -}; +void readPipeline(PipelineExecutor* executor, std::string json); +void addArrayReaders(PipelineExecutor* executor, std::vector arrays); +PyArrayObject* viewToNumpyArray(PointViewPtr view); +PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); +std::vector getArrays(const PipelineExecutor* executor); +std::vector getMeshes(const PipelineExecutor* executor); } // namespace python } // namespace pdal diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index e4fb31d6..911dcd7a 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -2,12 +2,13 @@ # cython: c_string_type=unicode, c_string_encoding=utf8 import json +from types import SimpleNamespace + from cpython.ref cimport Py_DECREF from libcpp.vector cimport vector from libcpp.string cimport string from libc.stdint cimport int64_t from libcpp cimport bool -from types import SimpleNamespace import numpy as np cimport numpy as np @@ -60,10 +61,9 @@ cdef extern from "PyArray.hpp" namespace "pdal::python": Array(np.ndarray) except + -cdef extern from "PyPipeline.hpp" namespace "pdal::python": - cdef cppclass PyPipelineExecutor: - PyPipelineExecutor(const char*) except + - PyPipelineExecutor(const char*, vector[Array*]&) except + +cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": + cdef cppclass PipelineExecutor: + PipelineExecutor(const char*) except + bool executed() except + int64_t execute() except + bool validate() except + @@ -71,23 +71,28 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": string getMetadata() except + string getSchema() except + string getLog() except + - vector[np.PyArrayObject*] getArrays() except + - vector[np.PyArrayObject*] getMeshes() except + int getLogLevel() void setLogLevel(int) +cdef extern from "PyPipeline.hpp" namespace "pdal::python": + void readPipeline(PipelineExecutor*, string) except + + void addArrayReaders(PipelineExecutor*, vector[Array *]) except + + vector[np.PyArrayObject*] getArrays(const PipelineExecutor* executor) except + + vector[np.PyArrayObject*] getMeshes(const PipelineExecutor* executor) except + + + cdef class Pipeline: - cdef PyPipelineExecutor* _executor + cdef PipelineExecutor* _executor cdef vector[Array *] _arrays; def __cinit__(self, unicode json, list arrays=None): + self._executor = new PipelineExecutor(json.encode('UTF-8')) + readPipeline(self._executor, json.encode('UTF-8')) if arrays is not None: for array in arrays: self._arrays.push_back(new Array(array)) - self._executor = new PyPipelineExecutor(json.encode('UTF-8'), self._arrays) - else: - self._executor = new PyPipelineExecutor(json.encode('UTF-8')) + addArrayReaders(self._executor, self._arrays) def __dealloc__(self): for array in self._arrays: @@ -120,13 +125,13 @@ cdef class Pipeline: def __get__(self): if not self._executor.executed(): raise RuntimeError("call execute() before fetching arrays") - return self._vector_to_list(self._executor.getArrays()) + return self._vector_to_list(getArrays(self._executor)) property meshes: def __get__(self): if not self._executor.executed(): raise RuntimeError("call execute() before fetching the mesh") - return self._vector_to_list(self._executor.getMeshes()) + return self._vector_to_list(getMeshes(self._executor)) def execute(self): return self._executor.execute() From 112bd5b8738c8e4e81199ec969b0de329d907f5f Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 5 Oct 2021 12:05:12 +0300 Subject: [PATCH 028/161] Refactor pipeline tests --- test/test_pipeline.py | 243 +++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 155 deletions(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 0113428c..c0179494 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -1,177 +1,130 @@ -import sys -import unittest -import pdal +import json import os -import numpy as np -from packaging.version import Version - -DATADIRECTORY = "./test/data" +import sys -bad_json = u""" -{ - "pipeline": [ - "nofile.las", - { - "type": "filters.sort", - "dimension": "X" - } - ] -} -""" +import numpy as np +import pytest +import pdal +DATADIRECTORY = os.path.join(os.path.dirname(__file__), "data") -class PDALTest(unittest.TestCase): - def fetch_json(self, filename): - import os - fn = DATADIRECTORY + os.path.sep + filename - output = '' - with open(fn, 'rb') as f: - output = f.read().decode('UTF-8') - return output +def get_pipeline(filename, factory=pdal.Pipeline): + with open(os.path.join(DATADIRECTORY, filename), "r") as f: + pipeline = factory(f.read()) + assert pipeline.validate() + return pipeline -class TestPipeline(PDALTest): - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), - "missing test data") +class TestPipeline: def test_construction(self): """Can we construct a PDAL pipeline""" - json = self.fetch_json('sort.json') - r = pdal.Pipeline(json) + assert isinstance(get_pipeline("sort.json"), pdal.Pipeline) - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), - "missing test data") def test_execution(self): """Can we execute a PDAL pipeline""" - x = self.fetch_json('sort.json') - r = pdal.Pipeline(x) - r.validate() + r = get_pipeline("sort.json") r.execute() - self.assertGreater(len(r.pipeline), 200) + assert len(r.pipeline) > 200 def test_validate(self): """Do we complain with bad pipelines""" + bad_json = """ + { + "pipeline": [ + "nofile.las", + { + "type": "filters.sort", + "dimension": "X" + } + ] + } + """ r = pdal.Pipeline(bad_json) - with self.assertRaises(RuntimeError): + with pytest.raises(RuntimeError): r.validate() - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), - "missing test data") def test_array(self): """Can we fetch PDAL data as a numpy array""" - json = self.fetch_json('sort.json') - r = pdal.Pipeline(json) - r.validate() + r = get_pipeline("sort.json") r.execute() arrays = r.arrays - self.assertEqual(len(arrays), 1) + assert len(arrays) == 1 a = arrays[0] - self.assertAlmostEqual(a[0][0], 635619.85, 7) - self.assertAlmostEqual(a[1064][2], 456.92, 7) + assert a[0][0] == 635619.85 + assert a[1064][2] == 456.92 - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), - "missing test data") def test_metadata(self): """Can we fetch PDAL metadata""" - json = self.fetch_json('sort.json') - r = pdal.Pipeline(json) - r.validate() + r = get_pipeline("sort.json") + with pytest.raises(RuntimeError): + r.metadata r.execute() - metadata = r.metadata - import json - j = json.loads(metadata) - self.assertEqual(j["metadata"]["readers.las"][0]["count"], 1065) + j = json.loads(r.metadata) + assert j["metadata"]["readers.las"][0]["count"] == 1065 + def test_schema(self): + """Fetching a schema works""" + r = get_pipeline("sort.json") + with pytest.raises(RuntimeError): + r.schema + r.execute() + assert r.schema["schema"]["dimensions"][0]["name"] == "X" - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), - "missing test data") def test_no_execute(self): """Does fetching arrays without executing throw an exception""" - json = self.fetch_json('sort.json') - r = pdal.Pipeline(json) - with self.assertRaises(RuntimeError): + r = get_pipeline("sort.json") + with pytest.raises(RuntimeError): r.arrays -# -# @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'reproject.json')), -# "missing test data") -# def test_logging(self): -# """Can we fetch log output""" -# json = self.fetch_json('reproject.json') -# r = pdal.Pipeline(json) -# r.loglevel = 8 -# r.validate() -# count = r.execute() -# self.assertEqual(count, 789) -# self.assertEqual(r.log.split()[0], '(pypipeline') -# - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), - "missing test data") - def test_schema(self): - """Fetching a schema works""" - json = self.fetch_json('sort.json') - r = pdal.Pipeline(json) - r.validate() - r.execute() - self.assertEqual(r.schema['schema']['dimensions'][0]['name'], 'X') - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'chip.json')), - "missing test data") def test_merged_arrays(self): - """Can we fetch multiple point views from merged PDAL data """ - json = self.fetch_json('chip.json') - r = pdal.Pipeline(json) - r.validate() + """Can we fetch multiple point views from merged PDAL data""" + r = get_pipeline("chip.json") r.execute() arrays = r.arrays - self.assertEqual(len(arrays), 43) + assert len(arrays) == 43 + # def test_logging(self): + # """Can we fetch log output""" + # r = get_pipeline('reproject.json') + # count = r.execute() + # assert count == 789 + # assert r.log.split()[0] == '(pypipeline') -class TestArrayLoad(PDALTest): - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'perlin.npy')), - "missing test data") +class TestArrayLoad: def test_merged_arrays(self): """Can we load data from a list of arrays to PDAL""" - if Version(pdal.info.version) < Version('1.8'): - return True - data = np.load(os.path.join(DATADIRECTORY, 'test3d.npy')) - + data = np.load(os.path.join(DATADIRECTORY, "test3d.npy")) arrays = [data, data, data] - - json = self.fetch_json('chip.json') - chip =u"""{ - "pipeline":[ - { - "type":"filters.range", - "limits":"Intensity[100:300)" - } - ] -}""" - - p = pdal.Pipeline(chip, arrays) - p.loglevel = 8 - count = p.execute() + filter_intensity = """{ + "pipeline":[ + { + "type":"filters.range", + "limits":"Intensity[100:300)" + } + ] + }""" + p = pdal.Pipeline(filter_intensity, arrays) + p.execute() arrays = p.arrays - self.assertEqual(len(arrays), 3) + assert len(arrays) == 3 for data in arrays: - self.assertEqual(len(data), 12) - self.assertEqual(data['Intensity'].sum(), 1926) + assert len(data) == 12 + assert data["Intensity"].sum() == 1926 def test_read_arrays(self): """Can we read and filter data from a list of arrays to PDAL""" - if Version(pdal.info.version) < Version('1.8'): - return True - # just some dummy data x_vals = [1.0, 2.0, 3.0, 4.0, 5.0] y_vals = [6.0, 7.0, 8.0, 9.0, 10.0] z_vals = [1.5, 3.5, 5.5, 7.5, 9.5] test_data = np.array( [(x, y, z) for x, y, z in zip(x_vals, y_vals, z_vals)], - dtype=[('X', np.float), ('Y', np.float), ('Z', np.float)] + dtype=[("X", np.float), ("Y", np.float), ("Z", np.float)], ) pipeline = """ @@ -184,19 +137,14 @@ def test_read_arrays(self): ] } """ - - p = pdal.Pipeline(pipeline, arrays=[test_data,]) - p.loglevel = 8 + p = pdal.Pipeline(pipeline, arrays=[test_data]) count = p.execute() arrays = p.arrays - self.assertEqual(count, 2) - self.assertEqual(len(arrays), 1) + assert count == 2 + assert len(arrays) == 1 def test_reference_counting(self): """Can we read and filter data from a list of arrays to PDAL""" - if Version(pdal.info.version) < Version("1.8"): - return True - # just some dummy data x_vals = [1.0, 2.0, 3.0, 4.0, 5.0] y_vals = [6.0, 7.0, 8.0, 9.0, 10.0] @@ -216,51 +164,36 @@ def test_reference_counting(self): ] } """ - p = pdal.Pipeline(pipeline, arrays=[test_data]) - p.loglevel = 8 count = p.execute() - self.assertEqual(count, 2) - self.assertEqual(1, sys.getrefcount(p.arrays[0]), "Reference count should only be 1 in this case") + assert count == 2 + refcount = sys.getrefcount(p.arrays[0]) + assert refcount == 1 -class TestDimensions(PDALTest): +class TestDimensions: def test_fetch_dimensions(self): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - self.assertLess(len(dims), 120) - self.assertGreater(len(dims), 71) + assert 71 < len(dims) < 120 -class TestMesh(PDALTest): - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), - "missing test data") + +class TestMesh: def test_no_execute(self): """Does fetching meshes without executing throw an exception""" - json = self.fetch_json('sort.json') - r = pdal.Pipeline(json) - with self.assertRaises(RuntimeError): + r = get_pipeline("sort.json") + with pytest.raises(RuntimeError): r.meshes - @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'mesh.json')), - "missing test data") def test_mesh(self): """Can we fetch PDAL face data as a numpy array""" - json = self.fetch_json('mesh.json') - r = pdal.Pipeline(json) - r.validate() + r = get_pipeline("mesh.json") points = r.execute() - self.assertEqual(points, 1065) + assert points == 1065 meshes = r.meshes - self.assertEqual(len(meshes), 24) + assert len(meshes) == 24 m = meshes[0] - self.assertEqual(str(m.dtype), "[('A', ' Date: Wed, 6 Oct 2021 11:56:09 +0300 Subject: [PATCH 029/161] Use mamba in place of conda for unix CI jobs --- .github/workflows/build.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 804e1acf..57cbafac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,8 +25,8 @@ jobs: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2 with: - auto-update-conda: true python-version: ${{ matrix.python-version }} + mamba-version: "*" activate-environment: test channels: conda-forge auto-activate-base: false @@ -34,13 +34,11 @@ jobs: - name: Linux dependencies if: matrix.os == 'ubuntu-latest' shell: bash -l {0} - run: | - sudo apt-get install gcc-multilib -y + run: sudo apt-get install gcc-multilib -y - name: Dependencies shell: bash -l {0} - run: | - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest - name: Install shell: bash -l {0} @@ -74,7 +72,6 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: channels: conda-forge - auto-update-conda: true python-version: ${{ matrix.python-version }} - name: Dependencies @@ -119,13 +116,12 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: channels: conda-forge - auto-update-conda: true python-version: ${{ matrix.python-version }} + mamba-version: "*" - name: Dependencies shell: bash -l {0} - run: | - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest - name: sdist shell: bash -l {0} From c1d03282e3cb5d2ee6e1b3a9c153089b3bb35565 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sun, 10 Oct 2021 17:43:15 +0300 Subject: [PATCH 030/161] Initial pure Python Pipeline, Stage and descendant classes --- pdal/__init__.py | 7 +- pdal/libpdalpython.pyx | 164 ++++++++++++++++++++------------ pdal/pipeline.py | 207 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 315 insertions(+), 63 deletions(-) create mode 100644 pdal/pipeline.py diff --git a/pdal/__init__.py b/pdal/__init__.py index 6ce05262..5d65114b 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,7 +1,8 @@ -from .libpdalpython import Pipeline -from .libpdalpython import getDimensions, getInfo - __version__ = "2.4.2" +__all__ = ["Pipeline", "Reader", "Filter", "Writer", "dimensions", "info"] + +from .libpdalpython import getDimensions, getInfo +from .pipeline import Pipeline, Reader, Filter, Writer dimensions = getDimensions() info = getInfo() diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 911dcd7a..638000b9 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -5,10 +5,9 @@ import json from types import SimpleNamespace from cpython.ref cimport Py_DECREF -from libcpp.vector cimport vector -from libcpp.string cimport string -from libc.stdint cimport int64_t from libcpp cimport bool +from libcpp.string cimport string +from libcpp.vector cimport vector import numpy as np cimport numpy as np @@ -56,6 +55,21 @@ def getDimensions(): return output +cdef extern from "pdal/StageFactory.hpp" namespace "pdal": + cpdef cppclass StageFactory: + @staticmethod + string inferReaderDriver(string) + @staticmethod + string inferWriterDriver(string) + + +def infer_reader_driver(driver): + return StageFactory.inferReaderDriver(driver) + +def infer_writer_driver(driver): + return StageFactory.inferWriterDriver(driver) + + cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: Array(np.ndarray) except + @@ -65,7 +79,7 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": cdef cppclass PipelineExecutor: PipelineExecutor(const char*) except + bool executed() except + - int64_t execute() except + + int execute() except + bool validate() except + string getPipeline() except + string getMetadata() except + @@ -78,66 +92,74 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": cdef extern from "PyPipeline.hpp" namespace "pdal::python": void readPipeline(PipelineExecutor*, string) except + void addArrayReaders(PipelineExecutor*, vector[Array *]) except + - vector[np.PyArrayObject*] getArrays(const PipelineExecutor* executor) except + - vector[np.PyArrayObject*] getMeshes(const PipelineExecutor* executor) except + + vector[np.PyArrayObject*] getArrays(const PipelineExecutor*) except + + vector[np.PyArrayObject*] getMeshes(const PipelineExecutor*) except + cdef class Pipeline: cdef PipelineExecutor* _executor - cdef vector[Array *] _arrays; - - def __cinit__(self, unicode json, list arrays=None): - self._executor = new PipelineExecutor(json.encode('UTF-8')) - readPipeline(self._executor, json.encode('UTF-8')) - if arrays is not None: - for array in arrays: - self._arrays.push_back(new Array(array)) - addArrayReaders(self._executor, self._arrays) + cdef vector[Array*] _inputs def __dealloc__(self): - for array in self._arrays: - del array - del self._executor - - property pipeline: - def __get__(self): - return self._executor.getPipeline() - - property metadata: - def __get__(self): - return self._executor.getMetadata() - - property loglevel: - def __get__(self): - return self._executor.getLogLevel() - def __set__(self, v): - self._executor.setLogLevel(v) - - property log: - def __get__(self): - return self._executor.getLog() - - property schema: - def __get__(self): - return json.loads(self._executor.getSchema()) - - property arrays: - def __get__(self): - if not self._executor.executed(): - raise RuntimeError("call execute() before fetching arrays") - return self._vector_to_list(getArrays(self._executor)) - - property meshes: - def __get__(self): - if not self._executor.executed(): - raise RuntimeError("call execute() before fetching the mesh") - return self._vector_to_list(getMeshes(self._executor)) + self.inputs = [] + + def __copy__(self): + cdef Pipeline clone = self.__class__() + clone._inputs = self._inputs + return clone + + @property + def inputs(self): + raise AttributeError("unreadable attribute") + + @inputs.setter + def inputs(self, ndarrays): + self._inputs.clear() + for ndarray in ndarrays: + self._inputs.push_back(new Array(ndarray)) + self._delete_executor() + + @property + def pipeline(self): + return self._get_executor().getPipeline() + + @property + def metadata(self): + return self._get_executor().getMetadata() + + @property + def loglevel(self): + return self._get_executor().getLogLevel() + + @loglevel.setter + def loglevel(self, level): + self._get_executor().setLogLevel(level) + + @property + def log(self): + return self._get_executor().getLog() + + @property + def schema(self): + return json.loads(self._get_executor().getSchema()) + + @property + def arrays(self): + if not self._get_executor().executed(): + raise RuntimeError("call execute() before fetching arrays") + return _vector_to_list(getArrays(self._executor)) + + @property + def meshes(self): + if not self._get_executor().executed(): + raise RuntimeError("call execute() before fetching the mesh") + return _vector_to_list(getMeshes(self._executor)) def execute(self): - return self._executor.execute() + return self._get_executor().execute() def validate(self): - return self._executor.validate() + return self._get_executor().validate() def get_meshio(self, idx): try: @@ -155,9 +177,31 @@ cdef class Pipeline: [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], ) - cdef _vector_to_list(self, vector[np.PyArrayObject*] arrays): - output = [] - for array in arrays: - output.append(array) - Py_DECREF(output[-1]) - return output + @property + def _json(self): + raise NotImplementedError("Abstract property") + + @property + def _num_inputs(self): + return self._inputs.size() + + def _delete_executor(self): + if self._executor: + del self._executor + self._executor = NULL + + cdef PipelineExecutor* _get_executor(self) except NULL: + if not self._executor: + json_bytes = self._json.encode("UTF-8") + self._executor = new PipelineExecutor(json_bytes) + readPipeline(self._executor, json_bytes) + addArrayReaders(self._executor, self._inputs) + return self._executor + + +cdef _vector_to_list(vector[np.PyArrayObject*] arrays): + output = [] + for array in arrays: + output.append(array) + Py_DECREF(output[-1]) + return output diff --git a/pdal/pipeline.py b/pdal/pipeline.py new file mode 100644 index 00000000..59295320 --- /dev/null +++ b/pdal/pipeline.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import glob +import json +import subprocess +from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast + +import numpy as np + +from . import libpdalpython + +_PDAL_DRIVERS = json.loads( + subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout +) + + +class Pipeline(libpdalpython.Pipeline): + def __init__( + self, + spec: Union[None, str, Sequence[Stage]] = None, + arrays: Sequence[np.ndarray] = (), + ): + self._stages: List[Stage] = [] + if spec: + stages = _parse_stages(spec) if isinstance(spec, str) else spec + for stage in stages: + self |= stage + if arrays: + self.inputs = arrays + + @property + def stages(self) -> List[Stage]: + return list(self._stages) + + def __ior__(self, other: Union[Stage, Pipeline]) -> Pipeline: + if isinstance(other, Stage): + self._stages.append(other) + elif isinstance(other, Pipeline): + if self._stages and other._num_inputs: + raise ValueError( + "A pipeline with inputs cannot follow another pipeline" + ) + self._stages.extend(other._stages) + else: + raise TypeError(f"Expected Stage or Pipeline, not {other}") + self._delete_executor() + return self + + def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: + new = self.__copy__() + new |= other + return new + + def __copy__(self) -> Pipeline: + clone = cast(Pipeline, super().__copy__()) + clone |= self + return clone + + @property + def _json(self) -> str: + options_list = [] + stage2tag: Dict[Stage, str] = {} + for stage in self._stages: + stage2tag[stage] = stage.tag or _generate_tag(stage, stage2tag.values()) + options = stage.options + options["tag"] = stage2tag[stage] + inputs = _get_input_tags(stage, stage2tag) + if inputs: + options["inputs"] = inputs + options_list.append(options) + return json.dumps(options_list) + + +class Stage: + def __init_subclass__(cls, type_prefix: str) -> None: + for driver in _PDAL_DRIVERS: + name = driver["name"] + prefix, _, suffix = name.partition(".") + if prefix == type_prefix: + cls._add_constructor(name, suffix, driver["description"]) + + def __init__(self, **options: Any): + self._options = options + + @property + def type(self) -> str: + return cast(str, self._options["type"]) + + @property + def tag(self) -> Optional[str]: + return self._options.get("tag") + + @property + def inputs(self) -> List[Union[Stage, str]]: + inputs = self._options.get("inputs", ()) + return [inputs] if isinstance(inputs, (Stage, str)) else list(inputs) + + @property + def options(self) -> Dict[str, Any]: + return dict(self._options) + + def pipeline(self, *arrays: np.ndarray) -> Pipeline: + return Pipeline((self,), arrays=arrays) + + def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: + return Pipeline((self, other)) + + @classmethod + def _add_constructor(cls, type: str, name: str, description: str) -> None: + constructor = lambda cls, *args, **kwargs: cls(*args, **kwargs, type=type) + constructor.__name__ = name + constructor.__qualname__ = f"{cls.__name__}.{name}" + constructor.__doc__ = description + setattr(cls, name, classmethod(constructor)) + + +class Reader(Stage, type_prefix="readers"): + def __init__(self, filename: str, **options: Any): + super().__init__(filename=filename, **options) + + @property + def type(self) -> str: + try: + return super().type + except KeyError: + filename = self._options["filename"] + return cast(str, libpdalpython.infer_reader_driver(filename)) + + +class Filter(Stage, type_prefix="filters"): + def __init__(self, type: str, **options: Any): + super().__init__(type=type, **options) + + +class Writer(Stage, type_prefix="writers"): + def __init__(self, filename: Optional[str] = None, **options: Any): + super().__init__(filename=filename, **options) + + @property + def type(self) -> str: + try: + return super().type + except KeyError: + filename = self._options["filename"] + return cast(str, libpdalpython.infer_writer_driver(filename)) + + +def _parse_stages(text: str) -> Iterator[Stage]: + json_stages = json.loads(text) + if isinstance(json_stages, dict): + json_stages = json_stages.get("pipeline") + if not isinstance(json_stages, list): + raise ValueError("root element is not a pipeline") + + last = len(json_stages) - 1 + for i, options in enumerate(json_stages): + if not isinstance(options, dict): + if isinstance(options, str): + options = {"filename": options} + else: + raise ValueError("A stage element must be string or dict") + + stage_type = options.get("type") + if stage_type: + is_reader = stage_type.startswith("readers.") + else: + # The type is inferred from a filename as a reader if it's not + # the last stage or if there's only one. + is_reader = i == 0 or i != last + + if is_reader: + paths = glob.glob(options.get("filename", "")) + if paths: + del options["filename"] + for path in paths: + yield Reader(filename=path, **options) + else: + yield Reader(**options) + elif not stage_type or stage_type.startswith("writers."): + yield Writer(**options) + else: + yield Filter(**options) + + +def _generate_tag(stage: Stage, tags: Container[str]) -> str: + tag_prefix = stage.type.replace(".", "_") + i = 1 + while True: + tag = tag_prefix + str(i) + if tag not in tags: + return tag + i += 1 + + +def _get_input_tags(stage: Stage, stage2tag: Dict[Stage, str]) -> List[str]: + tags = [] + for input in stage.inputs: + if isinstance(input, Stage): + try: + tags.append(stage2tag[input]) + except KeyError: + raise RuntimeError( + f"Invalid pipeline: Undefined stage " f"{input.tag or input.type!r}" + ) + else: + tags.append(input) + return tags From ea9e545690db5d6979a19a53f2c94498d0a50e7b Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 11 Oct 2021 13:53:08 +0300 Subject: [PATCH 031/161] Change Pipeline._inputs vector of raw Array pointers to shared_ptr to prevent mem leak on copy --- pdal/PyPipeline.cpp | 2 +- pdal/PyPipeline.hpp | 2 +- pdal/libpdalpython.pyx | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 0b488ccb..97c5b4fa 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -57,7 +57,7 @@ void readPipeline(PipelineExecutor* executor, std::string json) } -void addArrayReaders(PipelineExecutor* executor, std::vector arrays) +void addArrayReaders(PipelineExecutor* executor, std::vector> arrays) { // Make the symbols in pdal_base global so that they're accessible // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index b6d9c3f4..8628d013 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -46,7 +46,7 @@ namespace python class Array; void readPipeline(PipelineExecutor* executor, std::string json); -void addArrayReaders(PipelineExecutor* executor, std::vector arrays); +void addArrayReaders(PipelineExecutor* executor, std::vector> arrays); PyArrayObject* viewToNumpyArray(PointViewPtr view); PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); std::vector getArrays(const PipelineExecutor* executor); diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 638000b9..30e35fe7 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -6,6 +6,7 @@ from types import SimpleNamespace from cpython.ref cimport Py_DECREF from libcpp cimport bool +from libcpp.memory cimport make_shared, shared_ptr from libcpp.string cimport string from libcpp.vector cimport vector @@ -72,7 +73,7 @@ def infer_writer_driver(driver): cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: - Array(np.ndarray) except + + Array(np.PyArrayObject*) except + cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": @@ -91,14 +92,14 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": cdef extern from "PyPipeline.hpp" namespace "pdal::python": void readPipeline(PipelineExecutor*, string) except + - void addArrayReaders(PipelineExecutor*, vector[Array *]) except + + void addArrayReaders(PipelineExecutor*, vector[shared_ptr[Array]]) except + vector[np.PyArrayObject*] getArrays(const PipelineExecutor*) except + vector[np.PyArrayObject*] getMeshes(const PipelineExecutor*) except + cdef class Pipeline: cdef PipelineExecutor* _executor - cdef vector[Array*] _inputs + cdef vector[shared_ptr[Array]] _inputs def __dealloc__(self): self.inputs = [] @@ -116,7 +117,7 @@ cdef class Pipeline: def inputs(self, ndarrays): self._inputs.clear() for ndarray in ndarrays: - self._inputs.push_back(new Array(ndarray)) + self._inputs.push_back(make_shared[Array](ndarray)) self._delete_executor() @property From 0b31a609853db906f1797777b6023adedd2ef627 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 12 Oct 2021 17:49:31 +0300 Subject: [PATCH 032/161] Parametrize pipeline tests by filename and add copies of the test/data/*json pipelines as Python expressions using the new API --- test/data/bad.json | 9 +++++ test/data/bad.py | 1 + test/data/chip.py | 1 + test/data/mesh.py | 1 + test/data/reproject.py | 26 +++++++++++++ test/data/sort.py | 1 + test/test_pipeline.py | 84 ++++++++++++++++++++++-------------------- 7 files changed, 83 insertions(+), 40 deletions(-) create mode 100644 test/data/bad.json create mode 100644 test/data/bad.py create mode 100644 test/data/chip.py create mode 100644 test/data/mesh.py create mode 100644 test/data/reproject.py create mode 100644 test/data/sort.py diff --git a/test/data/bad.json b/test/data/bad.json new file mode 100644 index 00000000..13ee85cb --- /dev/null +++ b/test/data/bad.json @@ -0,0 +1,9 @@ +{ + "pipeline": [ + "nofile.las", + { + "type": "filters.sort", + "dimension": "X" + } + ] +} diff --git a/test/data/bad.py b/test/data/bad.py new file mode 100644 index 00000000..dfb3ec15 --- /dev/null +++ b/test/data/bad.py @@ -0,0 +1 @@ +Reader("nofile.las") | Filter.sort(dimension="X") diff --git a/test/data/chip.py b/test/data/chip.py new file mode 100644 index 00000000..6b229586 --- /dev/null +++ b/test/data/chip.py @@ -0,0 +1 @@ +Reader("test/data/autzen-utm.las") | Filter.chipper(capacity=25) | Writer("auzen-utm-chipped-25.las") diff --git a/test/data/mesh.py b/test/data/mesh.py new file mode 100644 index 00000000..635feea1 --- /dev/null +++ b/test/data/mesh.py @@ -0,0 +1 @@ +Reader("test/data/1.2-with-color.las") | Filter.splitter(length=1000) | Filter.delaunay() diff --git a/test/data/reproject.py b/test/data/reproject.py new file mode 100644 index 00000000..e7012d35 --- /dev/null +++ b/test/data/reproject.py @@ -0,0 +1,26 @@ +( + Reader(filename="test/data/1.2-with-color.las", spatialreference="EPSG:2993") + | + Filter.python(function="filter", module="anything", source=""" +import numpy as np + + +def filter(ins, outs): + cls = ins["Classification"] + keep_classes = [1] + + # Use the first test for our base array. + keep = np.equal(cls, keep_classes[0]) + + # For 1:n, test each predicate and join back + # to our existing predicate array + for k in range(1, len(keep_classes)): + t = np.equal(cls, keep_classes[k]) + keep = keep + t + + outs["Mask"] = keep + return True +""") + | + Writer("out2.las") +) diff --git a/test/data/sort.py b/test/data/sort.py new file mode 100644 index 00000000..8bc741a1 --- /dev/null +++ b/test/data/sort.py @@ -0,0 +1 @@ +Reader("test/data/1.2-with-color.las") | Filter.sort(dimension="X") diff --git a/test/test_pipeline.py b/test/test_pipeline.py index c0179494..f12161ee 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -10,44 +10,41 @@ DATADIRECTORY = os.path.join(os.path.dirname(__file__), "data") -def get_pipeline(filename, factory=pdal.Pipeline): +def get_pipeline(filename, validate=True): with open(os.path.join(DATADIRECTORY, filename), "r") as f: - pipeline = factory(f.read()) - assert pipeline.validate() + if filename.endswith(".json"): + pipeline = pdal.Pipeline(f.read()) + elif filename.endswith(".py"): + pipeline = eval(f.read(), vars(pdal)) + if validate: + assert pipeline.validate() return pipeline class TestPipeline: - def test_construction(self): + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_construction(self, filename): """Can we construct a PDAL pipeline""" - assert isinstance(get_pipeline("sort.json"), pdal.Pipeline) + assert isinstance(get_pipeline(filename), pdal.Pipeline) - def test_execution(self): + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_execution(self, filename): """Can we execute a PDAL pipeline""" - r = get_pipeline("sort.json") + r = get_pipeline(filename) r.execute() assert len(r.pipeline) > 200 - def test_validate(self): + @pytest.mark.parametrize("filename", ["bad.json", "bad.py"]) + def test_validate(self, filename): """Do we complain with bad pipelines""" - bad_json = """ - { - "pipeline": [ - "nofile.las", - { - "type": "filters.sort", - "dimension": "X" - } - ] - } - """ - r = pdal.Pipeline(bad_json) + r = get_pipeline(filename, validate=False) with pytest.raises(RuntimeError): r.validate() - def test_array(self): + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_array(self, filename): """Can we fetch PDAL data as a numpy array""" - r = get_pipeline("sort.json") + r = get_pipeline(filename) r.execute() arrays = r.arrays assert len(arrays) == 1 @@ -56,42 +53,47 @@ def test_array(self): assert a[0][0] == 635619.85 assert a[1064][2] == 456.92 - def test_metadata(self): + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_metadata(self, filename): """Can we fetch PDAL metadata""" - r = get_pipeline("sort.json") + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.metadata r.execute() j = json.loads(r.metadata) assert j["metadata"]["readers.las"][0]["count"] == 1065 - def test_schema(self): + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_schema(self, filename): """Fetching a schema works""" - r = get_pipeline("sort.json") + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.schema r.execute() assert r.schema["schema"]["dimensions"][0]["name"] == "X" - def test_no_execute(self): + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_no_execute(self, filename): """Does fetching arrays without executing throw an exception""" - r = get_pipeline("sort.json") + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.arrays - def test_merged_arrays(self): + @pytest.mark.parametrize("filename", ["chip.json", "chip.py"]) + def test_merged_arrays(self, filename): """Can we fetch multiple point views from merged PDAL data""" - r = get_pipeline("chip.json") + r = get_pipeline(filename) r.execute() arrays = r.arrays assert len(arrays) == 43 - # def test_logging(self): - # """Can we fetch log output""" - # r = get_pipeline('reproject.json') - # count = r.execute() - # assert count == 789 - # assert r.log.split()[0] == '(pypipeline') + @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) + def test_logging(self, filename): + """Can we fetch log output""" + r = get_pipeline(filename) + count = r.execute() + assert count == 789 + # assert r.log.split()[0] == "(pypipeline" class TestArrayLoad: @@ -179,15 +181,17 @@ def test_fetch_dimensions(self): class TestMesh: - def test_no_execute(self): + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_no_execute(self, filename): """Does fetching meshes without executing throw an exception""" - r = get_pipeline("sort.json") + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.meshes - def test_mesh(self): + @pytest.mark.parametrize("filename", ["mesh.json", "mesh.py"]) + def test_mesh(self, filename): """Can we fetch PDAL face data as a numpy array""" - r = get_pipeline("mesh.json") + r = get_pipeline(filename) points = r.execute() assert points == 1065 meshes = r.meshes From 06ce4115d2a62dbf5e3a1b6bbbf76bcc8d5a1404 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 12 Oct 2021 22:18:22 +0300 Subject: [PATCH 033/161] Add tests for building pipelines by piping stages together --- test/test_pipeline.py | 108 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 7 deletions(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index f12161ee..9114785e 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -21,12 +21,41 @@ def get_pipeline(filename, validate=True): return pipeline +def test_dimensions(): + """Ask PDAL for its valid dimensions list""" + dims = pdal.dimensions + assert 71 < len(dims) < 120 + + class TestPipeline: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_construction(self, filename): """Can we construct a PDAL pipeline""" assert isinstance(get_pipeline(filename), pdal.Pipeline) + # construct Pipeline from a sequence of stages + r = pdal.Reader("r") + f = pdal.Filter("f") + for spec in (r, f), [r, f]: + p = pdal.Pipeline(spec) + assert isinstance(p, pdal.Pipeline) + assert len(p.stages) == 2 + + @pytest.mark.parametrize( + "pipeline", + [ + "{}", + '{"foo": []}', + "[1, 2]", + '{"pipeline": [["a.las", "b.las"], "c.las"]}', + ], + ) + def test_invalid_json(self, pipeline): + """Do we complain with bad pipelines""" + json.loads(pipeline) + with pytest.raises(ValueError): + pdal.Pipeline(pipeline) + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_execution(self, filename): """Can we execute a PDAL pipeline""" @@ -87,6 +116,78 @@ def test_merged_arrays(self, filename): arrays = r.arrays assert len(arrays) == 43 + @pytest.mark.parametrize("filename", ["chip.json", "chip.py"]) + def test_stages(self, filename): + """Can we break up a pipeline as a sequence of stages""" + stages = pdal.Reader("test/data/autzen-utm.las").pipeline().stages + assert len(stages) == 1 + + stages = get_pipeline(filename).stages + assert len(stages) == 3 + + assert isinstance(stages[0], pdal.Reader) + assert stages[0].type == "readers.las" + + assert isinstance(stages[1], pdal.Filter) + assert stages[1].type == "filters.chipper" + + assert isinstance(stages[2], pdal.Writer) + assert stages[2].type == "writers.las" + + def test_pipe_stages(self): + """Can we build a pipeline by piping stages together""" + read = pdal.Reader("test/data/autzen-utm.las") + frange = pdal.Filter.range(limits="Intensity[50:200)") + fsplitter = pdal.Filter.splitter(length=1000) + fdelaunay = pdal.Filter.delaunay(inputs=[frange, fsplitter]) + + # pipe stages together + pipeline = read | frange | fsplitter | fdelaunay + assert pipeline.validate() + + # pipe a pipeline to a stage + pipeline = read | (frange | fsplitter | fdelaunay) + assert pipeline.validate() + + # pipe a pipeline to a pipeline + pipeline = (read | frange) | (fsplitter | fdelaunay) + assert pipeline.validate() + + def test_pipe_stage_errors(self): + """Do we complain with piping invalid objects""" + r = pdal.Reader("r", tag="r") + f = pdal.Filter("f") + w = pdal.Writer("w", inputs=["r", f]) + + with pytest.raises(TypeError): + r | (f, w) + with pytest.raises(TypeError): + (r, f) | w + with pytest.raises(TypeError): + (r, f) | (f, w) + + pipeline = r | w + with pytest.raises(RuntimeError) as ctx: + pipeline.validate() + assert "Undefined stage 'f'" in str(ctx.value) + + def test_inputs(self): + """Can we combine pipelines with inputs""" + data = np.load(os.path.join(DATADIRECTORY, "test3d.npy")) + f = pdal.Filter.splitter(length=1000) + pipeline = f.pipeline(data) + assert pipeline.validate() + + # a pipeline with inputs can be followed by stage/pipeline + assert (pipeline | pdal.Writer.null()).validate() + assert (pipeline | (f | pdal.Writer.null())).validate() + + # a pipeline with inputs cannot follow another stage/pipeline + with pytest.raises(ValueError): + pdal.Reader("r") | pipeline + with pytest.raises(ValueError): + (pdal.Reader("r") | f) | pipeline + @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) def test_logging(self, filename): """Can we fetch log output""" @@ -173,13 +274,6 @@ def test_reference_counting(self): assert refcount == 1 -class TestDimensions: - def test_fetch_dimensions(self): - """Ask PDAL for its valid dimensions list""" - dims = pdal.dimensions - assert 71 < len(dims) < 120 - - class TestMesh: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): From df7f161b6a0c23c8a543df6a2cd977038f1ab917 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 13 Oct 2021 15:15:12 +0300 Subject: [PATCH 034/161] Update README with the new Pipeline/Stage API --- README.rst | 131 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 76 insertions(+), 55 deletions(-) diff --git a/README.rst b/README.rst index dce0df7b..b695f025 100644 --- a/README.rst +++ b/README.rst @@ -62,6 +62,60 @@ sorts it by the ``X`` dimension: metadata = pipeline.metadata log = pipeline.log +Programmatic Pipeline Construction +................................................................................ + +The previous example specified the pipeline as a JSON string. Alternatively, a +pipeline can be constructed by creating ``Stage`` instances and piping them +together. For example, the previous pipeline can be specified as: + +.. code-block:: python + + pipeline = pdal.Reader("1.2-with-color.las") | pdal.Filter.sort(dimension="X") + +Stage Objects +============= + +- A stage is an instance of ``pdal.Reader``, ``pdal.Filter`` or ``pdal.Writer``. +- A stage can be instantiated by passing as keyword arguments the options + applicable to the respective PDAL stage. For more on PDAL stages and their + options, check the PDAL documentation on `Stage Objects `__. +- The ``filename`` option of ``Readers`` and ``Writers`` as well as the ``type`` + option of ``Filters`` can be passed positionally as the first argument. +- The ``inputs`` option specifies a sequence of stages to be set as input to the + current stage. Each input can be either the string tag of another stage, or + the ``Stage`` instance itself. +- The ``Reader``, ``Filter`` and ``Writer`` classes come with classmethods for + all the respective PDAL drivers. For example, ``pdal.Filter.head()`` is a + shortcut for ``pdal.Filter(type="filters.head")``. + +Pipeline Objects +================ + +A ``pdal.Pipeline`` instance can be created from: + +- a JSON string: ``Pipeline(json_string)`` +- a sequence of ``Stage`` instances: ``Pipeline([stage1, stage2])`` +- a single ``Stage`` with the ``Stage.pipeline`` method: ``stage.pipeline()`` +- nothing: ``Pipeline()`` creates a pipeline with no stages. +- joining ``Stage`` and/or other ``Pipeline`` instances together with the pipe + operator (``|``): + + - ``stage1 | stage2`` + - ``stage1 | pipeline1`` + - ``pipeline1 | stage1`` + - ``pipeline1 | pipeline2`` + +Every application of the pipe operator creates a new ``Pipeline`` instance. To +update an existing ``Pipeline`` use the respective in-place pipe operator (``|=``): + +.. code-block:: python + + # update pipeline in-place + pipeline = pdal.Pipeline() + pipeline |= stage + pipeline |= pipeline2 + Reading using Numpy Arrays ................................................................................ @@ -75,25 +129,15 @@ PDAL and Python: .. code-block:: python - data = "https://github.com/PDAL/PDAL/blob/master/test/data/las/1.2-with-color.las?raw=true" - - - json = """ - { - "pipeline": [ - { - "type": "readers.las", - "filename": "%s" - } - ] - }""" - import pdal import numpy as np - pipeline = pdal.Pipeline(json % data) - count = pipeline.execute() - # get the data from the first array + data = "https://github.com/PDAL/PDAL/blob/master/test/data/las/1.2-with-color.las?raw=true" + + pipeline = pdal.Reader.las(filename=data).pipeline() + print(pipeline.execute()) # 1065 points + + # Get the data from the first array # [array([(637012.24, 849028.31, 431.66, 143, 1, # 1, 1, 0, 1, -9., 132, 7326, 245380.78254963, 68, 77, 88), # dtype=[('X', ' 30] - print (len(intensity)) # 704 points + intensity = arr[arr["Intensity"] > 30] + print(len(intensity)) # 704 points - - # Now use pdal to clamp points that have intensity - # 100 <= v < 300, and there are 387 - clamp =u"""{ - "pipeline":[ - { - "type":"filters.range", - "limits":"Intensity[100:300)" - } - ] - }""" - - p = pdal.Pipeline(clamp, [intensity]) - count = p.execute() - clamped = p.arrays[0] - print (count) + # Now use pdal to clamp points that have intensity 100 <= v < 300 + pipeline = pdal.Filter.range(limits="Intensity[100:300)").pipeline(intensity) + print(pipeline.execute()) # 387 points + clamped = pipeline.arrays[0] # Write our intensity data to an LAS file - output =u"""{ - "pipeline":[ - { - "type":"writers.las", - "filename":"clamped.las", - "offset_x":"auto", - "offset_y":"auto", - "offset_z":"auto", - "scale_x":0.01, - "scale_y":0.01, - "scale_z":0.01 - } - ] - }""" - - p = pdal.Pipeline(output, [clamped]) - count = p.execute() - print (count) + pipeline = pdal.Writer.las( + filename="clamped2.las", + offset_x="auto", + offset_y="auto", + offset_z="auto", + scale_x=0.01, + scale_y=0.01, + scale_z=0.01, + ).pipeline(clamped) + print(pipeline.execute()) # 387 points Accessing Mesh Data From e568b468e332c60390a193b45f7775cdb656f15c Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 13 Oct 2021 17:42:49 +0300 Subject: [PATCH 035/161] Factor out Reader.type and Writer.type to a base InferableTypeStage class --- pdal/pipeline.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 59295320..6a8fa83e 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -3,6 +3,7 @@ import glob import json import subprocess +from abc import ABC, abstractmethod from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np @@ -72,7 +73,7 @@ def _json(self) -> str: class Stage: - def __init_subclass__(cls, type_prefix: str) -> None: + def __init_subclass__(cls, type_prefix: Optional[str] = None) -> None: for driver in _PDAL_DRIVERS: name = driver["name"] prefix, _, suffix = name.partition(".") @@ -114,17 +115,25 @@ def _add_constructor(cls, type: str, name: str, description: str) -> None: setattr(cls, name, classmethod(constructor)) -class Reader(Stage, type_prefix="readers"): - def __init__(self, filename: str, **options: Any): - super().__init__(filename=filename, **options) +class InferableTypeStage(ABC, Stage): + @staticmethod + @abstractmethod + def infer_type(filename: str) -> str: + """Infer the driver type from the filename""" @property def type(self) -> str: try: return super().type except KeyError: - filename = self._options["filename"] - return cast(str, libpdalpython.infer_reader_driver(filename)) + return self.infer_type(self._options["filename"]) + + +class Reader(InferableTypeStage, type_prefix="readers"): + infer_type = staticmethod(libpdalpython.infer_reader_driver) + + def __init__(self, filename: str, **options: Any): + super().__init__(filename=filename, **options) class Filter(Stage, type_prefix="filters"): @@ -132,18 +141,12 @@ def __init__(self, type: str, **options: Any): super().__init__(type=type, **options) -class Writer(Stage, type_prefix="writers"): +class Writer(InferableTypeStage, type_prefix="writers"): + infer_type = staticmethod(libpdalpython.infer_writer_driver) + def __init__(self, filename: Optional[str] = None, **options: Any): super().__init__(filename=filename, **options) - @property - def type(self) -> str: - try: - return super().type - except KeyError: - filename = self._options["filename"] - return cast(str, libpdalpython.infer_writer_driver(filename)) - def _parse_stages(text: str) -> Iterator[Stage]: json_stages = json.loads(text) From 7eb3be4c2cea44dec047ed6e459a08469c35c68f Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 14 Oct 2021 14:42:08 +0300 Subject: [PATCH 036/161] Refactor auto-injection of PDAL stage factory methods and include the respective options in their docstring --- pdal/__init__.py | 9 +++-- pdal/drivers.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ pdal/pipeline.py | 26 ++------------ 3 files changed, 97 insertions(+), 26 deletions(-) create mode 100644 pdal/drivers.py diff --git a/pdal/__init__.py b/pdal/__init__.py index 5d65114b..8b98b848 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,9 +1,12 @@ __version__ = "2.4.2" -__all__ = ["Pipeline", "Reader", "Filter", "Writer", "dimensions", "info"] +__all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] +from .drivers import inject_pdal_drivers from .libpdalpython import getDimensions, getInfo -from .pipeline import Pipeline, Reader, Filter, Writer +from .pipeline import Filter, Pipeline, Reader, Stage, Writer +inject_pdal_drivers() dimensions = getDimensions() info = getInfo() -del getDimensions, getInfo + +del inject_pdal_drivers, getDimensions, getInfo diff --git a/pdal/drivers.py b/pdal/drivers.py new file mode 100644 index 00000000..48276f7b --- /dev/null +++ b/pdal/drivers.py @@ -0,0 +1,88 @@ +import json +import subprocess +from dataclasses import dataclass, field +from typing import Callable, ClassVar, Mapping, Optional, Sequence, Type + +from .pipeline import Filter, Reader, Stage, Writer + + +@dataclass +class Option: + name: str + description: str + default: Optional[str] + + def __repr__(self) -> str: + if self.default is not None: + return f"{self.name}={self.default!r}: {self.description}" + else: + return f"{self.name}: {self.description}" + + +@dataclass +class Driver: + name: str + short_name: str = field(init=False) + type: Type[Stage] = field(init=False) + description: str + options: Optional[Sequence[Option]] + + def __post_init__(self) -> None: + prefix, _, suffix = self.name.partition(".") + self.type = self._prefix_to_type[prefix] + self.short_name = suffix + + @property + def factory(self) -> Callable[..., Stage]: + factory: Callable[..., Stage] + if self.type is Reader: + factory = lambda filename, **kwargs: Reader( + filename, type=self.name, **kwargs + ) + elif self.type is Writer: + factory = lambda filename=None, **kwargs: Writer( + filename, type=self.name, **kwargs + ) + else: + factory = lambda **kwargs: Filter(type=self.name, **kwargs) + factory.__name__ = self.short_name + factory.__qualname__ = f"{self.type.__name__}.{self.short_name}" + factory.__doc__ = self.description + if self.options: + factory.__doc__ += "\n\n" + factory.__doc__ += "\n".join(map(repr, self.options)) + return factory + + _prefix_to_type: ClassVar[Mapping[str, Type[Stage]]] = { + "readers": Reader, + "filters": Filter, + "writers": Writer, + } + + +def inject_pdal_drivers() -> None: + drivers = json.loads( + subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout + ) + options = dict( + json.loads( + subprocess.run( + ["pdal", "--options", "all", "--showjson"], capture_output=True + ).stdout + ) + ) + for d in drivers: + name = d["name"] + d_options = options.get(name) + if d_options is not None: + d_options = [ + Option(o["name"], o["description"], o.get("default")) for o in d_options + ] + # move filename option first + try: + i = next(i for i, opt in enumerate(d_options) if opt.name == "filename") + d_options.insert(0, d_options.pop(i)) + except StopIteration: + pass + driver = Driver(name, d["description"], d_options) + setattr(driver.type, driver.short_name, staticmethod(driver.factory)) diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 6a8fa83e..4658e216 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -2,7 +2,6 @@ import glob import json -import subprocess from abc import ABC, abstractmethod from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast @@ -10,10 +9,6 @@ from . import libpdalpython -_PDAL_DRIVERS = json.loads( - subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout -) - class Pipeline(libpdalpython.Pipeline): def __init__( @@ -73,13 +68,6 @@ def _json(self) -> str: class Stage: - def __init_subclass__(cls, type_prefix: Optional[str] = None) -> None: - for driver in _PDAL_DRIVERS: - name = driver["name"] - prefix, _, suffix = name.partition(".") - if prefix == type_prefix: - cls._add_constructor(name, suffix, driver["description"]) - def __init__(self, **options: Any): self._options = options @@ -106,14 +94,6 @@ def pipeline(self, *arrays: np.ndarray) -> Pipeline: def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return Pipeline((self, other)) - @classmethod - def _add_constructor(cls, type: str, name: str, description: str) -> None: - constructor = lambda cls, *args, **kwargs: cls(*args, **kwargs, type=type) - constructor.__name__ = name - constructor.__qualname__ = f"{cls.__name__}.{name}" - constructor.__doc__ = description - setattr(cls, name, classmethod(constructor)) - class InferableTypeStage(ABC, Stage): @staticmethod @@ -129,19 +109,19 @@ def type(self) -> str: return self.infer_type(self._options["filename"]) -class Reader(InferableTypeStage, type_prefix="readers"): +class Reader(InferableTypeStage): infer_type = staticmethod(libpdalpython.infer_reader_driver) def __init__(self, filename: str, **options: Any): super().__init__(filename=filename, **options) -class Filter(Stage, type_prefix="filters"): +class Filter(Stage): def __init__(self, type: str, **options: Any): super().__init__(type=type, **options) -class Writer(InferableTypeStage, type_prefix="writers"): +class Writer(InferableTypeStage): infer_type = staticmethod(libpdalpython.infer_writer_driver) def __init__(self, filename: Optional[str] = None, **options: Any): From 404821fd8f71990b98e399de73f2c25eb0aaac25 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 14 Oct 2021 16:03:16 +0300 Subject: [PATCH 037/161] Make filename optional for readers/writers --- pdal/drivers.py | 13 ++++--------- pdal/pipeline.py | 28 ++++++++++++---------------- test/test_pipeline.py | 9 +++++++++ 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pdal/drivers.py b/pdal/drivers.py index 48276f7b..22e961f8 100644 --- a/pdal/drivers.py +++ b/pdal/drivers.py @@ -34,17 +34,12 @@ def __post_init__(self) -> None: @property def factory(self) -> Callable[..., Stage]: - factory: Callable[..., Stage] - if self.type is Reader: - factory = lambda filename, **kwargs: Reader( - filename, type=self.name, **kwargs - ) - elif self.type is Writer: - factory = lambda filename=None, **kwargs: Writer( - filename, type=self.name, **kwargs + if self.options and self.options[0].name == "filename": + factory = lambda filename, **kwargs: self.type( + filename=filename, type=self.name, **kwargs ) else: - factory = lambda **kwargs: Filter(type=self.name, **kwargs) + factory = lambda **kwargs: self.type(type=self.name, **kwargs) factory.__name__ = self.short_name factory.__qualname__ = f"{self.type.__name__}.{self.short_name}" factory.__doc__ = self.description diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 4658e216..7511137a 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -2,7 +2,6 @@ import glob import json -from abc import ABC, abstractmethod from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np @@ -89,31 +88,31 @@ def options(self) -> Dict[str, Any]: return dict(self._options) def pipeline(self, *arrays: np.ndarray) -> Pipeline: - return Pipeline((self,), arrays=arrays) + return Pipeline((self,), arrays) def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return Pipeline((self, other)) -class InferableTypeStage(ABC, Stage): - @staticmethod - @abstractmethod - def infer_type(filename: str) -> str: - """Infer the driver type from the filename""" +class InferableTypeStage(Stage): + def __init__(self, filename: Optional[str] = None, **options: Any): + if filename: + options["filename"] = filename + super().__init__(**options) @property def type(self) -> str: try: return super().type except KeyError: - return self.infer_type(self._options["filename"]) + filename = self._options.get("filename") + return str(self._infer_type(filename) if filename else "") + _infer_type = staticmethod(lambda filename: "") -class Reader(InferableTypeStage): - infer_type = staticmethod(libpdalpython.infer_reader_driver) - def __init__(self, filename: str, **options: Any): - super().__init__(filename=filename, **options) +class Reader(InferableTypeStage): + _infer_type = staticmethod(libpdalpython.infer_reader_driver) class Filter(Stage): @@ -122,10 +121,7 @@ def __init__(self, type: str, **options: Any): class Writer(InferableTypeStage): - infer_type = staticmethod(libpdalpython.infer_writer_driver) - - def __init__(self, filename: Optional[str] = None, **options: Any): - super().__init__(filename=filename, **options) + _infer_type = staticmethod(libpdalpython.infer_writer_driver) def _parse_stages(text: str) -> Iterator[Stage]: diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 9114785e..dcce5fda 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -188,6 +188,15 @@ def test_inputs(self): with pytest.raises(ValueError): (pdal.Reader("r") | f) | pipeline + def test_infer_stage_type(self): + """Can we infer stage type from the filename""" + assert pdal.Reader("foo.las").type == "readers.las" + assert pdal.Writer("foo.las").type == "writers.las" + assert pdal.Reader("foo.xxx").type == "" + assert pdal.Writer("foo.xxx").type == "" + assert pdal.Reader().type == "" + assert pdal.Writer().type == "" + @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) def test_logging(self, filename): """Can we fetch log output""" From 765abc8ad547d4b77c793b73d8a00e72157913b0 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 14 Oct 2021 16:24:52 +0300 Subject: [PATCH 038/161] Update README noting that the auto-generated stage factories include the available options in their docstring --- README.rst | 33 ++++++++++++++++++++++++++------- pdal/drivers.py | 1 + 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index b695f025..cbe9ab34 100644 --- a/README.rst +++ b/README.rst @@ -80,14 +80,33 @@ Stage Objects - A stage can be instantiated by passing as keyword arguments the options applicable to the respective PDAL stage. For more on PDAL stages and their options, check the PDAL documentation on `Stage Objects `__. -- The ``filename`` option of ``Readers`` and ``Writers`` as well as the ``type`` - option of ``Filters`` can be passed positionally as the first argument. -- The ``inputs`` option specifies a sequence of stages to be set as input to the - current stage. Each input can be either the string tag of another stage, or - the ``Stage`` instance itself. -- The ``Reader``, ``Filter`` and ``Writer`` classes come with classmethods for + + - The ``filename`` option of ``Readers`` and ``Writers`` as well as the ``type`` + option of ``Filters`` can be passed positionally as the first argument. + - The ``inputs`` option specifies a sequence of stages to be set as input to the + current stage. Each input can be either the string tag of another stage, or + the ``Stage`` instance itself. +- The ``Reader``, ``Filter`` and ``Writer`` classes come with static methods for all the respective PDAL drivers. For example, ``pdal.Filter.head()`` is a - shortcut for ``pdal.Filter(type="filters.head")``. + shortcut for ``pdal.Filter(type="filters.head")``. These methods are + auto-generated by introspecting ``pdal`` and the available options are + included in each method's docstring: + +.. code-block:: + + >>> help(pdal.Filter.head) + Help on function head in module pdal.pipeline: + + head(**kwargs) + Return N points from beginning of the point cloud. + + user_data: User JSON + log: Debug output filename + option_file: File from which to read additional options + where: Expression describing points to be passed to this filter + where_merge='auto': If 'where' option is set, describes how skipped points should be merged with kept points in standard mode. + count='10': Number of points to return from beginning. If 'invert' is true, number of points to drop from the beginning. + invert='false': If true, 'count' specifies the number of points to skip from the beginning. Pipeline Objects ================ diff --git a/pdal/drivers.py b/pdal/drivers.py index 22e961f8..ce84985a 100644 --- a/pdal/drivers.py +++ b/pdal/drivers.py @@ -42,6 +42,7 @@ def factory(self) -> Callable[..., Stage]: factory = lambda **kwargs: self.type(type=self.name, **kwargs) factory.__name__ = self.short_name factory.__qualname__ = f"{self.type.__name__}.{self.short_name}" + factory.__module__ = self.type.__module__ factory.__doc__ = self.description if self.options: factory.__doc__ += "\n\n" From 0f408b760cc69183c148fe1cc4cc6541c0b74e96 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 14 Oct 2021 16:45:58 +0300 Subject: [PATCH 039/161] Simplify inject_pdal_drivers a bit --- pdal/drivers.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/pdal/drivers.py b/pdal/drivers.py index ce84985a..e314f030 100644 --- a/pdal/drivers.py +++ b/pdal/drivers.py @@ -10,7 +10,7 @@ class Option: name: str description: str - default: Optional[str] + default: Optional[str] = None def __repr__(self) -> str: if self.default is not None: @@ -25,7 +25,7 @@ class Driver: short_name: str = field(init=False) type: Type[Stage] = field(init=False) description: str - options: Optional[Sequence[Option]] + options: Sequence[Option] def __post_init__(self) -> None: prefix, _, suffix = self.name.partition(".") @@ -69,16 +69,12 @@ def inject_pdal_drivers() -> None: ) for d in drivers: name = d["name"] - d_options = options.get(name) - if d_options is not None: - d_options = [ - Option(o["name"], o["description"], o.get("default")) for o in d_options - ] - # move filename option first - try: - i = next(i for i, opt in enumerate(d_options) if opt.name == "filename") - d_options.insert(0, d_options.pop(i)) - except StopIteration: - pass + d_options = [Option(**option_dict) for option_dict in (options.get(name) or ())] + # move filename option first + try: + i = next(i for i, opt in enumerate(d_options) if opt.name == "filename") + d_options.insert(0, d_options.pop(i)) + except StopIteration: + pass driver = Driver(name, d["description"], d_options) setattr(driver.type, driver.short_name, staticmethod(driver.factory)) From 9ebc64fb8d9674146ed56c6c9e9631de4e6e2b2e Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 19 Oct 2021 19:22:58 +0300 Subject: [PATCH 040/161] Remove getArrays/getMeshes/_vector_to_list in place of executor.getManagerConst().views() --- pdal/PyPipeline.cpp | 17 ------------- pdal/PyPipeline.hpp | 2 -- pdal/libpdalpython.pyx | 57 +++++++++++++++++++++++++++++------------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 97c5b4fa..9fe2e9b2 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -177,14 +177,6 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view) return array; } -std::vector getArrays(const PipelineExecutor* executor) -{ - std::vector output; - for (auto view: executor->getManagerConst().views()) - output.push_back(viewToNumpyArray(view)); - return output; -} - PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) { @@ -237,14 +229,5 @@ PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) return array; } - -std::vector getMeshes(const PipelineExecutor* executor) -{ - std::vector output; - for (auto view: executor->getManagerConst().views()) - output.push_back(meshToNumpyArray(view->mesh())); - return output; -} - } // namespace python } // namespace pdal diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index 8628d013..aec9ef75 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -49,8 +49,6 @@ void readPipeline(PipelineExecutor* executor, std::string json); void addArrayReaders(PipelineExecutor* executor, std::vector> arrays); PyArrayObject* viewToNumpyArray(PointViewPtr view); PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); -std::vector getArrays(const PipelineExecutor* executor); -std::vector getMeshes(const PipelineExecutor* executor); } // namespace python } // namespace pdal diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 30e35fe7..735bf74f 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -4,9 +4,11 @@ import json from types import SimpleNamespace +from cython.operator cimport dereference as deref from cpython.ref cimport Py_DECREF from libcpp cimport bool from libcpp.memory cimport make_shared, shared_ptr +from libcpp.set cimport set as cpp_set from libcpp.string cimport string from libcpp.vector cimport vector @@ -71,14 +73,28 @@ def infer_writer_driver(driver): return StageFactory.inferWriterDriver(driver) -cdef extern from "PyArray.hpp" namespace "pdal::python": - cdef cppclass Array: - Array(np.PyArrayObject*) except + +cdef extern from "pdal/Mesh.hpp" namespace "pdal": + cdef cppclass TriangularMesh: + pass + + +cdef extern from "pdal/PointView.hpp" namespace "pdal": + cdef cppclass PointView: + TriangularMesh* mesh() + + ctypedef shared_ptr[PointView] PointViewPtr + ctypedef cpp_set[PointViewPtr] PointViewSet + + +cdef extern from "pdal/PipelineManager.hpp" namespace "pdal": + cdef cppclass PipelineManager: + const PointViewSet& views() const cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": cdef cppclass PipelineExecutor: PipelineExecutor(const char*) except + + const PipelineManager& getManagerConst() except + bool executed() except + int execute() except + bool validate() except + @@ -90,11 +106,16 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": void setLogLevel(int) +cdef extern from "PyArray.hpp" namespace "pdal::python": + cdef cppclass Array: + Array(np.PyArrayObject*) except + + + cdef extern from "PyPipeline.hpp" namespace "pdal::python": void readPipeline(PipelineExecutor*, string) except + void addArrayReaders(PipelineExecutor*, vector[shared_ptr[Array]]) except + - vector[np.PyArrayObject*] getArrays(const PipelineExecutor*) except + - vector[np.PyArrayObject*] getMeshes(const PipelineExecutor*) except + + np.PyArrayObject* viewToNumpyArray(PointViewPtr) except + + np.PyArrayObject* meshToNumpyArray(const TriangularMesh*) except + cdef class Pipeline: @@ -146,15 +167,25 @@ cdef class Pipeline: @property def arrays(self): - if not self._get_executor().executed(): + cdef PipelineExecutor* executor = self._get_executor() + if not executor.executed(): raise RuntimeError("call execute() before fetching arrays") - return _vector_to_list(getArrays(self._executor)) + output = [] + for view in executor.getManagerConst().views(): + output.append(viewToNumpyArray(view)) + Py_DECREF(output[-1]) + return output @property def meshes(self): - if not self._get_executor().executed(): + cdef PipelineExecutor* executor = self._get_executor() + if not executor.executed(): raise RuntimeError("call execute() before fetching the mesh") - return _vector_to_list(getMeshes(self._executor)) + output = [] + for view in executor.getManagerConst().views(): + output.append(meshToNumpyArray(deref(view).mesh())) + Py_DECREF(output[-1]) + return output def execute(self): return self._get_executor().execute() @@ -198,11 +229,3 @@ cdef class Pipeline: readPipeline(self._executor, json_bytes) addArrayReaders(self._executor, self._inputs) return self._executor - - -cdef _vector_to_list(vector[np.PyArrayObject*] arrays): - output = [] - for array in arrays: - output.append(array) - Py_DECREF(output[-1]) - return output From 2e816512876f28f5d609f62a52a59328340fe9cf Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 23 Oct 2021 13:12:18 +0300 Subject: [PATCH 041/161] fix logging --- pdal/libpdalpython.pyx | 12 +++++++----- pdal/pipeline.py | 25 +++++++++++++++++++++++++ test/data/reproject.json | 2 +- test/data/reproject.py | 2 ++ test/test_pipeline.py | 22 +++++++++++++++++++++- 5 files changed, 56 insertions(+), 7 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 735bf74f..0caaa491 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -102,8 +102,7 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": string getMetadata() except + string getSchema() except + string getLog() except + - int getLogLevel() - void setLogLevel(int) + void setLogLevel(int) except + cdef extern from "PyArray.hpp" namespace "pdal::python": @@ -121,6 +120,7 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef class Pipeline: cdef PipelineExecutor* _executor cdef vector[shared_ptr[Array]] _inputs + cdef int _loglevel def __dealloc__(self): self.inputs = [] @@ -151,11 +151,12 @@ cdef class Pipeline: @property def loglevel(self): - return self._get_executor().getLogLevel() + return self._loglevel @loglevel.setter - def loglevel(self, level): - self._get_executor().setLogLevel(level) + def loglevel(self, value): + self._loglevel = value + self._delete_executor() @property def log(self): @@ -226,6 +227,7 @@ cdef class Pipeline: if not self._executor: json_bytes = self._json.encode("UTF-8") self._executor = new PipelineExecutor(json_bytes) + self._executor.setLogLevel(self._loglevel) readPipeline(self._executor, json_bytes) addArrayReaders(self._executor, self._inputs) return self._executor diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 7511137a..084e07b0 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -2,6 +2,7 @@ import glob import json +import logging from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np @@ -9,11 +10,21 @@ from . import libpdalpython +LogLevelToPDAL = { + logging.ERROR: 0, + logging.WARNING: 1, + logging.INFO: 2, + logging.DEBUG: 8, # pdal::LogLevel::Debug5 +} +LogLevelFromPDAL = {v: k for k, v in LogLevelToPDAL.items()} + + class Pipeline(libpdalpython.Pipeline): def __init__( self, spec: Union[None, str, Sequence[Stage]] = None, arrays: Sequence[np.ndarray] = (), + loglevel: int = logging.ERROR, ): self._stages: List[Stage] = [] if spec: @@ -22,11 +33,25 @@ def __init__( self |= stage if arrays: self.inputs = arrays + self.loglevel = loglevel @property def stages(self) -> List[Stage]: return list(self._stages) + @property + def loglevel(self) -> int: + return LogLevelFromPDAL[super().loglevel] + + @loglevel.setter + def loglevel(self, value: int) -> None: + try: + loglevel = LogLevelToPDAL[value] + except KeyError: + raise ValueError(f"Invalid level {value!r}") + # super() property setter is not supported + libpdalpython.Pipeline.loglevel.__set__(self, loglevel) + def __ior__(self, other: Union[Stage, Pipeline]) -> Pipeline: if isinstance(other, Stage): self._stages.append(other) diff --git a/test/data/reproject.json b/test/data/reproject.json index 87cdc1b3..d0bd3ebd 100644 --- a/test/data/reproject.json +++ b/test/data/reproject.json @@ -7,7 +7,7 @@ { "type":"filters.python", "function":"filter", - "source":"import numpy as np\n\ndef filter(ins,outs):\n\tcls = ins['Classification']\n\n\tkeep_classes = [1]\n\n\t# Use the first test for our base array.\n\tkeep = np.equal(cls, keep_classes[0])\n\n\t# For 1:n, test each predicate and join back\n\t# to our existing predicate array\n\tfor k in range(1,len(keep_classes)):\n\t\tt = np.equal(cls, keep_classes[k])\n\t\tkeep = keep + t\n\n\touts['Mask'] = keep\n\treturn True", + "source":"import numpy as np\n\ndef filter(ins,outs):\n\tprint('entered filter()')\n\tcls = ins['Classification']\n\n\tkeep_classes = [1]\n\n\t# Use the first test for our base array.\n\tkeep = np.equal(cls, keep_classes[0])\n\n\t# For 1:n, test each predicate and join back\n\t# to our existing predicate array\n\tfor k in range(1,len(keep_classes)):\n\t\tt = np.equal(cls, keep_classes[k])\n\t\tkeep = keep + t\n\n\touts['Mask'] = keep\n\tprint('exiting filter()')\n\treturn True", "module":"anything" }, "out2.las" diff --git a/test/data/reproject.py b/test/data/reproject.py index e7012d35..e74514c0 100644 --- a/test/data/reproject.py +++ b/test/data/reproject.py @@ -6,6 +6,7 @@ def filter(ins, outs): + print("entered filter()") cls = ins["Classification"] keep_classes = [1] @@ -19,6 +20,7 @@ def filter(ins, outs): keep = keep + t outs["Mask"] = keep + print("exiting filter()") return True """) | diff --git a/test/test_pipeline.py b/test/test_pipeline.py index dcce5fda..4a07a538 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -1,4 +1,5 @@ import json +import logging import os import sys @@ -197,13 +198,32 @@ def test_infer_stage_type(self): assert pdal.Reader().type == "" assert pdal.Writer().type == "" + # fails against PDAL master; see https://github.com/PDAL/PDAL/issues/3566 + @pytest.mark.xfail @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) def test_logging(self, filename): """Can we fetch log output""" r = get_pipeline(filename) + assert r.loglevel == logging.ERROR + assert r.log == "" + + for loglevel in logging.CRITICAL, -1: + with pytest.raises(ValueError): + r.loglevel = loglevel + + count = r.execute() + assert count == 789 + assert r.log == "entered filter()\n" + "exiting filter()\n" + + r.loglevel = logging.DEBUG + assert r.loglevel == logging.DEBUG count = r.execute() assert count == 789 - # assert r.log.split()[0] == "(pypipeline" + assert "(pypipeline readers.las Debug)" in r.log + assert "(pypipeline filters.python Debug)" in r.log + assert "\nentered filter()\n" in r.log + assert "\nexiting filter()\n" in r.log + assert "(pypipeline writers.las Debug)" in r.log class TestArrayLoad: From de359ad75f43b3cd96c5f3bc9526997551d3b47d Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 23 Oct 2021 13:36:21 +0300 Subject: [PATCH 042/161] Make Pipeline._executor a unique_ptr --- pdal/libpdalpython.pyx | 74 +++++++++++++++++++++++------------------- pdal/pipeline.py | 7 ++-- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 0caaa491..2e20b304 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -7,7 +7,7 @@ from types import SimpleNamespace from cython.operator cimport dereference as deref from cpython.ref cimport Py_DECREF from libcpp cimport bool -from libcpp.memory cimport make_shared, shared_ptr +from libcpp.memory cimport make_shared, shared_ptr, unique_ptr from libcpp.set cimport set as cpp_set from libcpp.string cimport string from libcpp.vector cimport vector @@ -93,7 +93,7 @@ cdef extern from "pdal/PipelineManager.hpp" namespace "pdal": cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": cdef cppclass PipelineExecutor: - PipelineExecutor(const char*) except + + PipelineExecutor(string) except + const PipelineManager& getManagerConst() except + bool executed() except + int execute() except + @@ -118,7 +118,7 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef class Pipeline: - cdef PipelineExecutor* _executor + cdef unique_ptr[PipelineExecutor] _executor cdef vector[shared_ptr[Array]] _inputs cdef int _loglevel @@ -130,6 +130,8 @@ cdef class Pipeline: clone._inputs = self._inputs return clone + #========= writeable properties to be set before execution =========================== + @property def inputs(self): raise AttributeError("unreadable attribute") @@ -139,15 +141,7 @@ cdef class Pipeline: self._inputs.clear() for ndarray in ndarrays: self._inputs.push_back(make_shared[Array](ndarray)) - self._delete_executor() - - @property - def pipeline(self): - return self._get_executor().getPipeline() - - @property - def metadata(self): - return self._get_executor().getMetadata() + self._del_executor() @property def loglevel(self): @@ -156,7 +150,9 @@ cdef class Pipeline: @loglevel.setter def loglevel(self, value): self._loglevel = value - self._delete_executor() + self._del_executor() + + #========= readable properties to be read after execution ============================ @property def log(self): @@ -166,6 +162,14 @@ cdef class Pipeline: def schema(self): return json.loads(self._get_executor().getSchema()) + @property + def pipeline(self): + return self._get_executor().getPipeline() + + @property + def metadata(self): + return self._get_executor().getMetadata() + @property def arrays(self): cdef PipelineExecutor* executor = self._get_executor() @@ -188,18 +192,13 @@ cdef class Pipeline: Py_DECREF(output[-1]) return output - def execute(self): - return self._get_executor().execute() - - def validate(self): - return self._get_executor().validate() - def get_meshio(self, idx): try: from meshio import Mesh except ModuleNotFoundError: raise RuntimeError( - "The get_meshio function can only be used if you have installed meshio. Try pip install meshio" + "The get_meshio function can only be used if you have installed meshio. " + "Try pip install meshio" ) array = self.arrays[idx] mesh = self.meshes[idx] @@ -210,24 +209,33 @@ cdef class Pipeline: [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], ) + #========= validation & execution methods ============================================ + + def validate(self): + return self._get_executor(set_if_unset=True).validate() + + def execute(self): + return self._get_executor(set_if_unset=True).execute() + + #========= non-public properties & methods =========================================== + @property def _json(self): raise NotImplementedError("Abstract property") @property - def _num_inputs(self): - return self._inputs.size() + def _has_inputs(self): + return not self._inputs.empty() - def _delete_executor(self): - if self._executor: - del self._executor - self._executor = NULL + def _del_executor(self): + self._executor.reset() - cdef PipelineExecutor* _get_executor(self) except NULL: - if not self._executor: + cdef PipelineExecutor* _get_executor(self, bool set_if_unset=False) except NULL: + if not self._executor and set_if_unset: json_bytes = self._json.encode("UTF-8") - self._executor = new PipelineExecutor(json_bytes) - self._executor.setLogLevel(self._loglevel) - readPipeline(self._executor, json_bytes) - addArrayReaders(self._executor, self._inputs) - return self._executor + executor = new PipelineExecutor(json_bytes) + executor.setLogLevel(self._loglevel) + readPipeline(executor, json_bytes) + addArrayReaders(executor, self._inputs) + self._executor.reset(executor) + return self._executor.get() diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 084e07b0..eb8fe4d3 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -31,8 +31,7 @@ def __init__( stages = _parse_stages(spec) if isinstance(spec, str) else spec for stage in stages: self |= stage - if arrays: - self.inputs = arrays + self.inputs = arrays self.loglevel = loglevel @property @@ -56,14 +55,14 @@ def __ior__(self, other: Union[Stage, Pipeline]) -> Pipeline: if isinstance(other, Stage): self._stages.append(other) elif isinstance(other, Pipeline): - if self._stages and other._num_inputs: + if self._stages and other._has_inputs: raise ValueError( "A pipeline with inputs cannot follow another pipeline" ) self._stages.extend(other._stages) else: raise TypeError(f"Expected Stage or Pipeline, not {other}") - self._delete_executor() + self._del_executor() return self def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: From 9383e27ea4922558c83be9ca01ed5745de0aeb2f Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 23 Oct 2021 21:27:50 +0300 Subject: [PATCH 043/161] Move libpdalpython.Pipeline.get_meshio to pure Python Pipeline --- .github/workflows/build.yml | 6 +++--- pdal/libpdalpython.pyx | 18 ------------------ pdal/pipeline.py | 19 +++++++++++++++++++ test/test_pipeline.py | 9 +++++++++ 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57cbafac..318aa784 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: - name: Dependencies shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest meshio - name: Install shell: bash -l {0} @@ -78,7 +78,7 @@ jobs: shell: cmd /C CALL "{0}" run: | call conda activate test - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest + conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest meshio - name: Install shell: cmd /C CALL "{0}" @@ -121,7 +121,7 @@ jobs: - name: Dependencies shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal - name: sdist shell: bash -l {0} diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 2e20b304..f4d2fa91 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -12,7 +12,6 @@ from libcpp.set cimport set as cpp_set from libcpp.string cimport string from libcpp.vector cimport vector -import numpy as np cimport numpy as np np.import_array() @@ -192,23 +191,6 @@ cdef class Pipeline: Py_DECREF(output[-1]) return output - def get_meshio(self, idx): - try: - from meshio import Mesh - except ModuleNotFoundError: - raise RuntimeError( - "The get_meshio function can only be used if you have installed meshio. " - "Try pip install meshio" - ) - array = self.arrays[idx] - mesh = self.meshes[idx] - if len(mesh) == 0: - return None - return Mesh( - np.stack((array["X"], array["Y"], array["Z"]), 1), - [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], - ) - #========= validation & execution methods ============================================ def validate(self): diff --git a/pdal/pipeline.py b/pdal/pipeline.py index eb8fe4d3..e7be4a34 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -6,6 +6,10 @@ from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np +try: + from meshio import Mesh +except ModuleNotFoundError: + Mesh = None from . import libpdalpython @@ -75,6 +79,21 @@ def __copy__(self) -> Pipeline: clone |= self return clone + def get_meshio(self, idx: int) -> Mesh: + if Mesh is None: + raise RuntimeError( + "The get_meshio function can only be used if you have installed meshio. " + "Try pip install meshio" + ) + array = self.arrays[idx] + mesh = self.meshes[idx] + if len(mesh) == 0: + return None + return Mesh( + np.stack((array["X"], array["Y"], array["Z"]), 1), + [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], + ) + @property def _json(self) -> str: options_list = [] diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 4a07a538..97b0ddf7 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -324,3 +324,12 @@ def test_mesh(self, filename): assert str(m.dtype) == "[('A', ' Date: Mon, 25 Oct 2021 14:30:02 +0300 Subject: [PATCH 044/161] Move json import out of libpdalpython.pyx --- pdal/libpdalpython.pyx | 3 +-- pdal/pipeline.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index f4d2fa91..a2444b75 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -1,7 +1,6 @@ # distutils: language = c++ # cython: c_string_type=unicode, c_string_encoding=utf8 -import json from types import SimpleNamespace from cython.operator cimport dereference as deref @@ -159,7 +158,7 @@ cdef class Pipeline: @property def schema(self): - return json.loads(self._get_executor().getSchema()) + return self._get_executor().getSchema() @property def pipeline(self): diff --git a/pdal/pipeline.py b/pdal/pipeline.py index e7be4a34..2f2c8aa8 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -42,6 +42,10 @@ def __init__( def stages(self) -> List[Stage]: return list(self._stages) + @property + def schema(self) -> Any: + return json.loads(super().schema) + @property def loglevel(self) -> int: return LogLevelFromPDAL[super().loglevel] From 01bc05ea57438116ea5e226cbc1173d89e8d1bb3 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 10:41:45 +0300 Subject: [PATCH 045/161] Drop set_if_unset_ parameter from Pipeline._get_executor --- pdal/libpdalpython.pyx | 8 +++---- test/test_pipeline.py | 50 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index a2444b75..44fb2db9 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -193,10 +193,10 @@ cdef class Pipeline: #========= validation & execution methods ============================================ def validate(self): - return self._get_executor(set_if_unset=True).validate() + return self._get_executor().validate() def execute(self): - return self._get_executor(set_if_unset=True).execute() + return self._get_executor().execute() #========= non-public properties & methods =========================================== @@ -211,8 +211,8 @@ cdef class Pipeline: def _del_executor(self): self._executor.reset() - cdef PipelineExecutor* _get_executor(self, bool set_if_unset=False) except NULL: - if not self._executor and set_if_unset: + cdef PipelineExecutor* _get_executor(self) except NULL: + if not self._executor: json_bytes = self._json.encode("UTF-8") executor = new PipelineExecutor(json_bytes) executor.setLogLevel(self._loglevel) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 97b0ddf7..b882dec0 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -86,9 +86,14 @@ def test_array(self, filename): @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_metadata(self, filename): """Can we fetch PDAL metadata""" + r = get_pipeline(filename, validate=False) + with pytest.raises(RuntimeError): + r.metadata + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.metadata + r.execute() j = json.loads(r.metadata) assert j["metadata"]["readers.las"][0]["count"] == 1065 @@ -96,15 +101,52 @@ def test_metadata(self, filename): @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_schema(self, filename): """Fetching a schema works""" + r = get_pipeline(filename, validate=False) + with pytest.raises(RuntimeError): + r.schema + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.schema + r.execute() assert r.schema["schema"]["dimensions"][0]["name"] == "X" + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_pipeline(self, filename): + """Can we fetch PDAL pipeline string""" + r = get_pipeline(filename, validate=False) + with pytest.raises(RuntimeError): + r.pipeline + + r = get_pipeline(filename) + with pytest.raises(RuntimeError): + r.pipeline + + r.execute() + assert json.loads(r.pipeline) == { + "pipeline": [ + { + "filename": "test/data/1.2-with-color.las", + "tag": "readers_las1", + "type": "readers.las", + }, + { + "dimension": "X", + "inputs": ["readers_las1"], + "tag": "filters_sort1", + "type": "filters.sort", + }, + ] + } + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): """Does fetching arrays without executing throw an exception""" + r = get_pipeline(filename, validate=False) + with pytest.raises(RuntimeError): + r.arrays + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.arrays @@ -203,6 +245,10 @@ def test_infer_stage_type(self): @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) def test_logging(self, filename): """Can we fetch log output""" + r = get_pipeline(filename, validate=False) + assert r.loglevel == logging.ERROR + assert r.log == "" + r = get_pipeline(filename) assert r.loglevel == logging.ERROR assert r.log == "" @@ -307,6 +353,10 @@ class TestMesh: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): """Does fetching meshes without executing throw an exception""" + r = get_pipeline(filename, validate=False) + with pytest.raises(RuntimeError): + r.meshes + r = get_pipeline(filename) with pytest.raises(RuntimeError): r.meshes From 11e5cb995067d61bb624e11d000362839ba93be7 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 15:52:53 +0300 Subject: [PATCH 046/161] Drop Pipeline.validate(); deprecated since https://github.com/PDAL/python/issues/46 --- pdal/libpdalpython.pyx | 12 +++-------- test/test_pipeline.py | 48 ++++++++++-------------------------------- 2 files changed, 14 insertions(+), 46 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 44fb2db9..bb743dc0 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -95,7 +95,6 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": const PipelineManager& getManagerConst() except + bool executed() except + int execute() except + - bool validate() except + string getPipeline() except + string getMetadata() except + string getSchema() except + @@ -128,6 +127,9 @@ cdef class Pipeline: clone._inputs = self._inputs return clone + def execute(self): + return self._get_executor().execute() + #========= writeable properties to be set before execution =========================== @property @@ -190,14 +192,6 @@ cdef class Pipeline: Py_DECREF(output[-1]) return output - #========= validation & execution methods ============================================ - - def validate(self): - return self._get_executor().validate() - - def execute(self): - return self._get_executor().execute() - #========= non-public properties & methods =========================================== @property diff --git a/test/test_pipeline.py b/test/test_pipeline.py index b882dec0..33ffd59e 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -11,14 +11,12 @@ DATADIRECTORY = os.path.join(os.path.dirname(__file__), "data") -def get_pipeline(filename, validate=True): +def get_pipeline(filename): with open(os.path.join(DATADIRECTORY, filename), "r") as f: if filename.endswith(".json"): pipeline = pdal.Pipeline(f.read()) elif filename.endswith(".py"): pipeline = eval(f.read(), vars(pdal)) - if validate: - assert pipeline.validate() return pipeline @@ -67,9 +65,9 @@ def test_execution(self, filename): @pytest.mark.parametrize("filename", ["bad.json", "bad.py"]) def test_validate(self, filename): """Do we complain with bad pipelines""" - r = get_pipeline(filename, validate=False) + r = get_pipeline(filename) with pytest.raises(RuntimeError): - r.validate() + r.execute() @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_array(self, filename): @@ -86,25 +84,17 @@ def test_array(self, filename): @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_metadata(self, filename): """Can we fetch PDAL metadata""" - r = get_pipeline(filename, validate=False) - with pytest.raises(RuntimeError): - r.metadata - r = get_pipeline(filename) with pytest.raises(RuntimeError): r.metadata r.execute() j = json.loads(r.metadata) - assert j["metadata"]["readers.las"][0]["count"] == 1065 + assert j["metadata"]["readers.las"]["count"] == 1065 @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_schema(self, filename): """Fetching a schema works""" - r = get_pipeline(filename, validate=False) - with pytest.raises(RuntimeError): - r.schema - r = get_pipeline(filename) with pytest.raises(RuntimeError): r.schema @@ -115,10 +105,6 @@ def test_schema(self, filename): @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_pipeline(self, filename): """Can we fetch PDAL pipeline string""" - r = get_pipeline(filename, validate=False) - with pytest.raises(RuntimeError): - r.pipeline - r = get_pipeline(filename) with pytest.raises(RuntimeError): r.pipeline @@ -143,10 +129,6 @@ def test_pipeline(self, filename): @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): """Does fetching arrays without executing throw an exception""" - r = get_pipeline(filename, validate=False) - with pytest.raises(RuntimeError): - r.arrays - r = get_pipeline(filename) with pytest.raises(RuntimeError): r.arrays @@ -186,15 +168,15 @@ def test_pipe_stages(self): # pipe stages together pipeline = read | frange | fsplitter | fdelaunay - assert pipeline.validate() + pipeline.execute() # pipe a pipeline to a stage pipeline = read | (frange | fsplitter | fdelaunay) - assert pipeline.validate() + pipeline.execute() # pipe a pipeline to a pipeline pipeline = (read | frange) | (fsplitter | fdelaunay) - assert pipeline.validate() + pipeline.execute() def test_pipe_stage_errors(self): """Do we complain with piping invalid objects""" @@ -211,7 +193,7 @@ def test_pipe_stage_errors(self): pipeline = r | w with pytest.raises(RuntimeError) as ctx: - pipeline.validate() + pipeline.execute() assert "Undefined stage 'f'" in str(ctx.value) def test_inputs(self): @@ -219,11 +201,11 @@ def test_inputs(self): data = np.load(os.path.join(DATADIRECTORY, "test3d.npy")) f = pdal.Filter.splitter(length=1000) pipeline = f.pipeline(data) - assert pipeline.validate() + pipeline.execute() # a pipeline with inputs can be followed by stage/pipeline - assert (pipeline | pdal.Writer.null()).validate() - assert (pipeline | (f | pdal.Writer.null())).validate() + (pipeline | pdal.Writer.null()).execute() + (pipeline | (f | pdal.Writer.null())).execute() # a pipeline with inputs cannot follow another stage/pipeline with pytest.raises(ValueError): @@ -245,10 +227,6 @@ def test_infer_stage_type(self): @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) def test_logging(self, filename): """Can we fetch log output""" - r = get_pipeline(filename, validate=False) - assert r.loglevel == logging.ERROR - assert r.log == "" - r = get_pipeline(filename) assert r.loglevel == logging.ERROR assert r.log == "" @@ -353,10 +331,6 @@ class TestMesh: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): """Does fetching meshes without executing throw an exception""" - r = get_pipeline(filename, validate=False) - with pytest.raises(RuntimeError): - r.meshes - r = get_pipeline(filename) with pytest.raises(RuntimeError): r.meshes From 138589113fb1b523d0616e016b6b33bed8c9c30b Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 1 Nov 2021 10:01:34 +0200 Subject: [PATCH 047/161] Misc tweaks --- .github/workflows/build.yml | 12 ++++-------- pdal/CMakeLists.txt | 18 ++---------------- pdal/__init__.py | 8 ++++---- pdal/libpdalpython.pyx | 22 ++++++++-------------- pdal/pipeline.py | 19 ++++++++++--------- 5 files changed, 28 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 318aa784..4e877586 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,19 +43,17 @@ jobs: - name: Install shell: bash -l {0} run: | - python setup.py install -- -G "Ninja" -DWITH_TESTS=ON + python setup.py develop -- -G "Ninja" -DWITH_TESTS=ON pdal --drivers - name: Test shell: bash -l {0} run: | - pip install -e . SKPATH=$(python -c "import skbuild; print (skbuild.constants.SKBUILD_DIR())")/cmake-build export PDAL_DRIVER_PATH=$SKPATH - ls $SKPATH $SKPATH/pdal_filters_python_test $SKPATH/pdal_io_numpy_test - py.test + py.test -v windows: name: ${{ matrix.os }} - ${{ matrix.python-version }} @@ -84,21 +82,19 @@ jobs: shell: cmd /C CALL "{0}" run: | call conda activate test - where python python setup.py develop -- -G "Ninja" -DWITH_TESTS=ON + pdal --drivers - name: Test shell: cmd /C CALL "{0}" run: | call conda activate test set PYTHONHOME=%CONDA_PREFIX% - py.test for /f %%i in ('python -c "import skbuild; print (skbuild.constants.SKBUILD_DIR())"') do set SKPATH=%%i set PDAL_DRIVER_PATH=%SKPATH%\cmake-build %SKPATH%\cmake-build\pdal_filters_python_test.exe %SKPATH%\cmake-build\pdal_io_numpy_test.exe - pdal --version - pdal --drivers + py.test -v dist: name: Distribution diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt index 97bc051a..86ef1671 100644 --- a/pdal/CMakeLists.txt +++ b/pdal/CMakeLists.txt @@ -1,24 +1,10 @@ -set(EXTENSION_SRC - PyArray.cpp - PyArray.hpp - PyDimension.hpp - PyPipeline.cpp - PyPipeline.hpp) +set(EXTENSION_SRC PyArray.cpp PyPipeline.cpp) set(extension "libpdalpython") add_cython_target(${extension} "libpdalpython.pyx" CXX PY3) - add_library(${extension} MODULE ${EXTENSION_SRC} libpdalpython) -target_include_directories( ${extension} - PRIVATE - . - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS}) - +target_include_directories(${extension} PRIVATE . ${Python3_NumPy_INCLUDE_DIRS}) target_link_libraries(${extension} ${PDAL_LIBRARIES}) python_extension_module(${extension}) install(TARGETS ${extension} LIBRARY DESTINATION "pdal") - - diff --git a/pdal/__init__.py b/pdal/__init__.py index 8b98b848..09d58be9 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,12 +1,12 @@ __version__ = "2.4.2" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] +from . import libpdalpython from .drivers import inject_pdal_drivers -from .libpdalpython import getDimensions, getInfo from .pipeline import Filter, Pipeline, Reader, Stage, Writer inject_pdal_drivers() -dimensions = getDimensions() -info = getInfo() +dimensions = libpdalpython.getDimensions() +info = libpdalpython.getInfo() -del inject_pdal_drivers, getDimensions, getInfo +del inject_pdal_drivers, libpdalpython diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index bb743dc0..5da576da 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -119,14 +119,6 @@ cdef class Pipeline: cdef vector[shared_ptr[Array]] _inputs cdef int _loglevel - def __dealloc__(self): - self.inputs = [] - - def __copy__(self): - cdef Pipeline clone = self.__class__() - clone._inputs = self._inputs - return clone - def execute(self): return self._get_executor().execute() @@ -194,23 +186,25 @@ cdef class Pipeline: #========= non-public properties & methods =========================================== - @property - def _json(self): - raise NotImplementedError("Abstract property") + def _get_json(self): + raise NotImplementedError("Abstract method") @property def _has_inputs(self): return not self._inputs.empty() + def _copy_inputs(self, Pipeline other): + self._inputs = other._inputs + def _del_executor(self): self._executor.reset() cdef PipelineExecutor* _get_executor(self) except NULL: if not self._executor: - json_bytes = self._json.encode("UTF-8") - executor = new PipelineExecutor(json_bytes) + json = self._get_json() + executor = new PipelineExecutor(json) executor.setLogLevel(self._loglevel) - readPipeline(executor, json_bytes) + readPipeline(executor, json) addArrayReaders(executor, self._inputs) self._executor.reset(executor) return self._executor.get() diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 2f2c8aa8..19b26ade 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -6,14 +6,14 @@ from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np + try: from meshio import Mesh -except ModuleNotFoundError: +except ModuleNotFoundError: # pragma: no cover Mesh = None from . import libpdalpython - LogLevelToPDAL = { logging.ERROR: 0, logging.WARNING: 1, @@ -30,6 +30,7 @@ def __init__( arrays: Sequence[np.ndarray] = (), loglevel: int = logging.ERROR, ): + super().__init__() self._stages: List[Stage] = [] if spec: stages = _parse_stages(spec) if isinstance(spec, str) else spec @@ -79,12 +80,13 @@ def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return new def __copy__(self) -> Pipeline: - clone = cast(Pipeline, super().__copy__()) + clone = self.__class__(loglevel=self.loglevel) + clone._copy_inputs(self) clone |= self return clone - def get_meshio(self, idx: int) -> Mesh: - if Mesh is None: + def get_meshio(self, idx: int) -> Optional[Mesh]: + if Mesh is None: # pragma: no cover raise RuntimeError( "The get_meshio function can only be used if you have installed meshio. " "Try pip install meshio" @@ -98,8 +100,7 @@ def get_meshio(self, idx: int) -> Mesh: [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], ) - @property - def _json(self) -> str: + def _get_json(self) -> str: options_list = [] stage2tag: Dict[Stage, str] = {} for stage in self._stages: @@ -134,8 +135,8 @@ def inputs(self) -> List[Union[Stage, str]]: def options(self) -> Dict[str, Any]: return dict(self._options) - def pipeline(self, *arrays: np.ndarray) -> Pipeline: - return Pipeline((self,), arrays) + def pipeline(self, *arrays: np.ndarray, loglevel: int = logging.ERROR) -> Pipeline: + return Pipeline((self,), arrays, loglevel) def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return Pipeline((self, other)) From 72e6cb4e1960a1d2c42edef5ae4dbc716815ae81 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 9 Nov 2021 15:37:07 +0200 Subject: [PATCH 048/161] Make addArrayReaders a no-op for empty arrays --- pdal/PyPipeline.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 9fe2e9b2..0aa9e8d4 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -69,6 +69,8 @@ void addArrayReaders(PipelineExecutor* executor, std::vectorgetManager(); std::vector roots = manager.roots(); From 150affe97b2a87ac43f6c24a57713a92b524c205 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 9 Nov 2021 16:38:29 +0200 Subject: [PATCH 049/161] Pass filename to PDAL as is, don't expand the glob in Python --- pdal/pipeline.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 19b26ade..c95bb81a 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -1,6 +1,5 @@ from __future__ import annotations -import glob import json import logging from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast @@ -196,13 +195,7 @@ def _parse_stages(text: str) -> Iterator[Stage]: is_reader = i == 0 or i != last if is_reader: - paths = glob.glob(options.get("filename", "")) - if paths: - del options["filename"] - for path in paths: - yield Reader(filename=path, **options) - else: - yield Reader(**options) + yield Reader(**options) elif not stage_type or stage_type.startswith("writers."): yield Writer(**options) else: From 86a33eaed2af4892dcbe099018564393c40acf0b Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 9 Nov 2021 22:51:02 +0200 Subject: [PATCH 050/161] Append a merge filter to pipelines that consist of readers only --- pdal/pipeline.py | 5 ++++- test/test_pipeline.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pdal/pipeline.py b/pdal/pipeline.py index c95bb81a..a9ad42dc 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -102,7 +102,10 @@ def get_meshio(self, idx: int) -> Optional[Mesh]: def _get_json(self) -> str: options_list = [] stage2tag: Dict[Stage, str] = {} - for stage in self._stages: + stages = self._stages + if all(isinstance(stage, Reader) for stage in stages): + stages = [*stages, Filter.merge()] + for stage in stages: stage2tag[stage] = stage.tag or _generate_tag(stage, stage2tag.values()) options = stage.options options["tag"] = stage2tag[stage] diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 33ffd59e..8b04c2ef 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -249,6 +249,20 @@ def test_logging(self, filename): assert "\nexiting filter()\n" in r.log assert "(pypipeline writers.las Debug)" in r.log + def test_only_readers(self): + """Does a pipeline that consists of only readers return the merged data""" + read = pdal.Reader("test/data/*.las") + r1 = read.pipeline() + count1 = r1.execute() + array1 = r1.arrays[0] + + r2 = read | read + count2 = r2.execute() + array2 = r2.arrays[0] + + assert count2 == 2 * count1 + np.testing.assert_array_equal(np.concatenate([array1, array1]), array2) + class TestArrayLoad: def test_merged_arrays(self): From 4effb524691fb46dba634f103b0360d0f808b63d Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 10 Nov 2021 23:40:31 +0200 Subject: [PATCH 051/161] Update test_pipeline to pass on Windows until https://github.com/PDAL/PDAL/issues/3574 is fixed --- test/test_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 8b04c2ef..3fc85ca2 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -113,7 +113,8 @@ def test_pipeline(self, filename): assert json.loads(r.pipeline) == { "pipeline": [ { - "filename": "test/data/1.2-with-color.las", + # TODO: update this after https://github.com/PDAL/PDAL/issues/3574 + "filename": f"test/data{os.sep}1.2-with-color.las", "tag": "readers_las1", "type": "readers.las", }, From 3829ad50d8cc89d12a8f28b5d6d2ffcf40b86cbc Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 13 Nov 2021 12:41:42 +0200 Subject: [PATCH 052/161] Add streamable property to Stage and Pipeline --- pdal/drivers.py | 9 ++++++++- pdal/pipeline.py | 10 +++++++++- test/test_pipeline.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/pdal/drivers.py b/pdal/drivers.py index e314f030..54ea8820 100644 --- a/pdal/drivers.py +++ b/pdal/drivers.py @@ -1,10 +1,12 @@ import json import subprocess from dataclasses import dataclass, field -from typing import Callable, ClassVar, Mapping, Optional, Sequence, Type +from typing import Callable, ClassVar, FrozenSet, Mapping, Optional, Sequence, Type from .pipeline import Filter, Reader, Stage, Writer +StreamableTypes: FrozenSet + @dataclass class Option: @@ -67,6 +69,7 @@ def inject_pdal_drivers() -> None: ).stdout ) ) + streamable = [] for d in drivers: name = d["name"] d_options = [Option(**option_dict) for option_dict in (options.get(name) or ())] @@ -78,3 +81,7 @@ def inject_pdal_drivers() -> None: pass driver = Driver(name, d["description"], d_options) setattr(driver.type, driver.short_name, staticmethod(driver.factory)) + if d["streamable"]: + streamable.append(driver.name) + global StreamableTypes + StreamableTypes = frozenset(streamable) diff --git a/pdal/pipeline.py b/pdal/pipeline.py index a9ad42dc..ef581e98 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -11,7 +11,7 @@ except ModuleNotFoundError: # pragma: no cover Mesh = None -from . import libpdalpython +from . import drivers, libpdalpython LogLevelToPDAL = { logging.ERROR: 0, @@ -42,6 +42,10 @@ def __init__( def stages(self) -> List[Stage]: return list(self._stages) + @property + def streamable(self) -> bool: + return all(stage.streamable for stage in self._stages) + @property def schema(self) -> Any: return json.loads(super().schema) @@ -124,6 +128,10 @@ def __init__(self, **options: Any): def type(self) -> str: return cast(str, self._options["type"]) + @property + def streamable(self) -> bool: + return self.type in drivers.StreamableTypes + @property def tag(self) -> Optional[str]: return self._options.get("tag") diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 3fc85ca2..e0b7d691 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -223,6 +223,41 @@ def test_infer_stage_type(self): assert pdal.Reader().type == "" assert pdal.Writer().type == "" + def test_streamable(self): + """Can we distinguish streamable from non-streamable stages and pipeline""" + rs = pdal.Reader(type="readers.las", filename="foo") + assert rs.streamable is True + assert pdal.Reader.las("foo").streamable is True + assert pdal.Reader("foo.las").streamable is True + + rn = pdal.Reader(type="readers.pts", filename="foo") + assert rn.streamable is False + assert pdal.Reader.pts("foo").streamable is False + assert pdal.Reader("foo.pts").streamable is False + + fs = pdal.Filter(type="filters.crop") + assert fs.streamable is True + assert pdal.Filter.crop().streamable is True + + fn = pdal.Filter(type="filters.cluster") + assert fn.streamable is False + assert pdal.Filter.cluster().streamable is False + + ws = pdal.Writer(type="writers.ogr", filename="foo") + assert ws.streamable is True + assert pdal.Writer.ogr(filename="foo").streamable is True + assert pdal.Writer("foo.shp").streamable is True + + wn = pdal.Writer(type="writers.glb", filename="foo") + assert wn.streamable is False + assert pdal.Writer.gltf("foo").streamable is False + assert pdal.Writer("foo.glb").streamable is False + + assert (rs | fs | ws).streamable is True + assert (rn | fs | ws).streamable is False + assert (rs | fn | ws).streamable is False + assert (rs | fs | wn).streamable is False + # fails against PDAL master; see https://github.com/PDAL/PDAL/issues/3566 @pytest.mark.xfail @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) From 6f017ca1e446ebc52bd0a9a2a017345c65534ac8 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Tue, 19 Oct 2021 10:58:32 -0400 Subject: [PATCH 053/161] Interface using numpy array as point table and streaming. - Needs to handle skips - Need to check on error handling with premature termination, etc. --- pdal/CMakeLists.txt | 2 +- pdal/StreamableExecutor.cpp | 308 ++++++++++++++++++++++++++++++++++++ pdal/StreamableExecutor.hpp | 112 +++++++++++++ pdal/__init__.py | 12 +- pdal/libpdalpython.pyx | 38 +++++ test/data/range.json | 7 + test/test_pipeline.py | 40 +++++ 7 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 pdal/StreamableExecutor.cpp create mode 100644 pdal/StreamableExecutor.hpp create mode 100644 test/data/range.json diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt index 86ef1671..04ea5730 100644 --- a/pdal/CMakeLists.txt +++ b/pdal/CMakeLists.txt @@ -1,4 +1,4 @@ -set(EXTENSION_SRC PyArray.cpp PyPipeline.cpp) +set(EXTENSION_SRC PyArray.cpp PyPipeline.cpp StreamableExecutor.cpp) set(extension "libpdalpython") add_cython_target(${extension} "libpdalpython.pyx" CXX PY3) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp new file mode 100644 index 00000000..c37f7c8b --- /dev/null +++ b/pdal/StreamableExecutor.cpp @@ -0,0 +1,308 @@ +/****************************************************************************** +* Copyright (c) 2016, Howard Butler (howard@hobu.co) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "StreamableExecutor.hpp" + +#include +#include + +#include +#include + +namespace pdal +{ +namespace python +{ + +// PythonPointTable + +PythonPointTable::PythonPointTable(point_count_t limit, int prefetch) : + StreamPointTable(m_layout, limit), m_limit(limit), m_prefetch(prefetch), + m_curArray(nullptr), m_dtype(nullptr) +{} + +PythonPointTable::~PythonPointTable() +{ + py_destroy(); +} + +void PythonPointTable::finalize() +{ + BasePointTable::finalize(); + py_createDescriptor(); + m_curArray = py_createArray(); +} + +void PythonPointTable::py_destroy() +{ + auto gil = PyGILState_Ensure(); + + Py_XDECREF(m_dtype); + Py_XDECREF(m_curArray); + + PyGILState_Release(gil); +} + +PyArrayObject *PythonPointTable::py_createArray() const +{ + auto gil = PyGILState_Ensure(); + + npy_intp size = (npy_intp)m_limit; + Py_INCREF(m_dtype); + PyArrayObject *arr = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, m_dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); + + PyGILState_Release(gil); + return arr; +} + +void PythonPointTable::py_createDescriptor() +{ + auto gil = PyGILState_Ensure(); + + if (_import_array() < 0) + std::cerr << "Could not import array!\n"; + PyObject *dtype_dict = py_buildNumpyDescriptor(); + if (PyArray_DescrConverter(dtype_dict, &m_dtype) == NPY_FAIL) + m_dtype = nullptr; + Py_XDECREF(dtype_dict); + + PyGILState_Release(gil); +} + +void PythonPointTable::py_resizeArray(int np) +{ + npy_intp sizes[1]; + sizes[0] = np; + PyArray_Dims dims{ sizes, 1 }; + + auto gil = PyGILState_Ensure(); + PyArray_Resize(m_curArray, &dims, true, NPY_CORDER); + PyGILState_Release(gil); +} + +PyObject *PythonPointTable::py_buildNumpyDescriptor() const +{ + // Build up a numpy dtype dictionary + // + // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', + // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], + // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', + // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', + // 'Classification', 'ScanAngleRank', 'UserData', + // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} + // + + DimTypeList dims = layout()->dimTypes(); + PyObject* names = PyList_New(dims.size()); + PyObject* formats = PyList_New(dims.size()); + for (size_t i = 0; i < dims.size(); ++i) + { + DimType& dt = dims[i]; + std::string name = m_layout.dimName(dt.m_id); + npy_intp stride = Dimension::size(dt.m_type); + + std::string kind; + Dimension::BaseType b = Dimension::base(dt.m_type); + if (b == Dimension::BaseType::Unsigned) + kind = "u"; + else if (b == Dimension::BaseType::Signed) + kind = "i"; + else if (b == Dimension::BaseType::Floating) + kind = "f"; + else + throw pdal_error("Unable to map kind '" + kind + + "' to PDAL dimension type"); + + std::string type = kind + std::to_string(stride); + PyList_SetItem(names, i, PyUnicode_FromString(name.c_str())); + PyList_SetItem(formats, i, PyUnicode_FromString(type.c_str())); + } + + PyObject* dict = PyDict_New(); + PyDict_SetItemString(dict, "names", names); + PyDict_SetItemString(dict, "formats", formats); + return dict; +} + +void PythonPointTable::reset() +{ + point_count_t np = numPoints(); + + // If this is the last chunk, the size might be less than what's expected, so + // resize the array to match its true size. + // ABELL - This isn't quite right. We want to know if there are any skips and deal with those + // but I'm leaving that for the moment. + if (np && np != m_limit) + py_resizeArray(np); + + // This will keep putting arrays on the list until done, whether or not the consumer + // can handle them that fast. We can modify as appropriate to block if desired. + std::unique_lock l(m_mutex); + { + // It's possible that this is called with 0 points processed, in which case + // we don't push the current array. + if (np) + { + m_arrays.push(m_curArray); + m_curArray = nullptr; + } + + bool done = np < m_limit; + + // If we just pushed the last chunk, push a nullptr so that a reader knows. + if (done) + m_arrays.push(nullptr); + m_producedCv.notify_one(); + + if (done) + return; + + while (m_arrays.size() > m_prefetch) + m_consumedCv.wait(l); + } + + // Make a new array for data. + m_curArray = py_createArray(); +} + +PyArrayObject *PythonPointTable::fetchArray() +{ + PyArrayObject *arr = nullptr; + + // Lock scope. + Py_BEGIN_ALLOW_THREADS + { + std::unique_lock l(m_mutex); + while (m_arrays.empty()) + m_producedCv.wait(l); + + // Grab the array from the front of the list and notify that we did so. + arr = m_arrays.front(); + m_arrays.pop(); + } + Py_END_ALLOW_THREADS + // Notify that we consumed an array. + m_consumedCv.notify_one(); + return arr; +} + +char *PythonPointTable::getPoint(PointId idx) +{ + return (char *)PyArray_GETPTR1(m_curArray, (npy_intp)idx); +} + + +// StreamableExecutor + +StreamableExecutor::StreamableExecutor(const char *json, int chunkSize, int prefetch) : + m_json(json), m_table(chunkSize, prefetch), m_manager(chunkSize), + m_log(Log::makeLog("pdal_python", &m_logStream)) +{ + m_manager.setLog(m_log); +} + +StreamableExecutor::~StreamableExecutor() +{ + //ABELL - Hmmm. + if (m_thread) + m_thread->join(); +} + +PyArrayObject *StreamableExecutor::executeNext() +{ + if (!m_thread) + { + m_thread.reset(new std::thread([this]() + { + m_manager.executeStream(m_table); + })); + } + + // Blocks until something is ready. + PyArrayObject *arr = m_table.fetchArray(); + if (arr == nullptr) + { + Py_BEGIN_ALLOW_THREADS + m_thread->join(); + Py_END_ALLOW_THREADS + m_thread.reset(); + } + return arr; +} + +std::string StreamableExecutor::getMetadata() const +{ + return pdal::Utils::toJSON(m_manager.getMetadata().clone("metadata")); +} + +std::string StreamableExecutor::getPipeline() const +{ + std::stringstream strm; + pdal::PipelineWriter::writePipeline(m_manager.getStage(), strm); + return strm.str(); +} + +int StreamableExecutor::getLogLevel() const +{ + return static_cast(m_log->getLevel()); +} + +std::string StreamableExecutor::getLog() const +{ + return m_logStream.str(); +} + +// Returns the active log level. +int StreamableExecutor::setLogLevel(int level) +{ + if (level < 0 || level > 8) + return getLogLevel(); + + m_log->setLevel(static_cast(level)); + return level; +} + +bool StreamableExecutor::validate() +{ + std::stringstream strm; + strm << m_json; + m_manager.readPipeline(strm); + m_manager.prepare(); + + return true; +} + +} // namespace python +} // namespace pdal diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp new file mode 100644 index 00000000..06a94d25 --- /dev/null +++ b/pdal/StreamableExecutor.hpp @@ -0,0 +1,112 @@ +/****************************************************************************** +* Copyright (c) 2016, Howard Butler (howard@hobu.co) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace pdal +{ +namespace python +{ + +class PythonPointTable : public StreamPointTable +{ +public: + PythonPointTable(point_count_t size, int prefetch); + ~PythonPointTable(); + + virtual void finalize(); + PyArrayObject *fetchArray(); + +protected: + virtual void reset(); + virtual char *getPoint(PointId idx); + +private: + // All functions starting with py_ call Python things that need the GIL locked. + void py_destroy(); + PyArrayObject *py_createArray() const; + void py_createDescriptor(); + void py_resizeArray(int np); + PyObject *py_buildNumpyDescriptor() const; + + point_count_t m_limit; + int m_prefetch; + PointLayout m_layout; + PyArrayObject *m_curArray; + PyArray_Descr *m_dtype; + std::mutex m_mutex; + std::condition_variable m_producedCv; + std::condition_variable m_consumedCv; + std::queue m_arrays; +}; + +class StreamableExecutor +{ +public: + StreamableExecutor(const char *json, int chunkSize, int prefetch); + ~StreamableExecutor(); + + PyArrayObject *executeNext(); + std::string getMetadata() const; + std::string getPipeline() const; + std::string getSchema() const; + std::string getLog() const; + int getLogLevel() const; + int setLogLevel(int level); + bool validate(); + +private: + std::string m_json; + int m_prefetch; + PythonPointTable m_table; + PipelineManager m_manager; + std::stringstream m_logStream; + LogPtr m_log; + std::unique_ptr m_thread; +}; + +} // namespace python +} // namespace pdal diff --git a/pdal/__init__.py b/pdal/__init__.py index 09d58be9..74037412 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,8 +1,18 @@ __version__ = "2.4.2" -__all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] +__all__ = [ + "Pipeline", + "PipelineIterator", + "Stage", + "Reader", + "Filter", + "Writer", + "dimensions", + "info", +] from . import libpdalpython from .drivers import inject_pdal_drivers +from .libpdalpython import PipelineIterator from .pipeline import Filter, Pipeline, Reader, Stage, Writer inject_pdal_drivers() diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 5da576da..12ad505b 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -113,6 +113,44 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": np.PyArrayObject* viewToNumpyArray(PointViewPtr) except + np.PyArrayObject* meshToNumpyArray(const TriangularMesh*) except + +cdef extern from "StreamableExecutor.hpp" namespace "pdal::python": + cdef cppclass StreamableExecutor: + StreamableExecutor(const char *, int, int) except + + np.PyArrayObject *executeNext() except + + bool validate() except + + string getMetadata() except + + string getPipeline() except + + string getSchema() except + + string getLog() except + + int getLogLevel() except + + int setLogLevel(int level) except + + + +cdef class PipelineIterator: + cdef int _chunk_size + cdef StreamableExecutor *_executor; + + def __cinit__(self, unicode json, list arrays=None, int chunk_size=10000, int prefetch=0): + self._chunk_size + self._executor = new StreamableExecutor(json.encode("UTF-8"), chunk_size, prefetch) +# self._add_arrays(arrays) + + property chunk_size: + def __get__(self): + return self._chunk_size + + def validate(self): + return self._executor.validate() + + def __iter__(self): + while True: + arrPtr = self._executor.executeNext() + if arrPtr is NULL: + break + arr = arrPtr + Py_DECREF(arr) + yield arr + cdef class Pipeline: cdef unique_ptr[PipelineExecutor] _executor diff --git a/test/data/range.json b/test/data/range.json new file mode 100644 index 00000000..df48ac09 --- /dev/null +++ b/test/data/range.json @@ -0,0 +1,7 @@ +[ + "test/data/autzen-utm.las", + { + "type": "filters.range", + "limits": "Intensity[80:120)" + } +] diff --git a/test/test_pipeline.py b/test/test_pipeline.py index e0b7d691..30dce928 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -2,6 +2,7 @@ import logging import os import sys +from functools import partial import numpy as np import pytest @@ -19,6 +20,12 @@ def get_pipeline(filename): pipeline = eval(f.read(), vars(pdal)) return pipeline +def get_pipeline_iter(filename, chunk_size=10000): + with open(os.path.join(DATADIRECTORY, filename), "r") as f: + pipeline = pdal.PipelineIterator(f.read(), chunk_size=chunk_size, prefetch=3) + assert pipeline.validate() + return pipeline + def test_dimensions(): """Ask PDAL for its valid dimensions list""" @@ -407,3 +414,36 @@ def test_meshio(self, filename): triangles = mesh.cells_dict["triangle"] assert len(triangles) == 134 assert triangles[0][0] == 29 + + +class TestPipelineIterator: + + def test_construction(self): + """Can we construct a PDAL pipeline iterator""" + assert isinstance( + get_pipeline_iter("range.json"), pdal.PipelineIterator + ) + assert isinstance( + get_pipeline_iter("range.json", chunk_size=100), pdal.PipelineIterator + ) + + def test_validate(self): + """Do we complain with bad pipelines""" + bad_json = """ + [ + "nofile.las", + { + "type": "filters.range", + "limits": "Intensity[80:120)" + } + ] + """ + r = pdal.PipelineIterator(bad_json) + with pytest.raises(RuntimeError): + r.validate() + + def test_iterate(self): + pipeline = get_pipeline_iter("range.json") + print("Got into iterate!") + for arr in pipeline: + print(arr.shape) From 6c5271ec65dbec6d1c0b35e7584234ef3439922f Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 20 Oct 2021 17:11:33 +0300 Subject: [PATCH 054/161] Inherit StreamableExecutor from PipelineExecutor --- pdal/PyPipeline.cpp | 7 --- pdal/PyPipeline.hpp | 1 - pdal/StreamableExecutor.cpp | 52 ++--------------- pdal/StreamableExecutor.hpp | 18 ++---- pdal/__init__.py | 12 +--- pdal/libpdalpython.pyx | 96 ++++++++++++++++--------------- pdal/pipeline.py | 4 ++ test/test_pipeline.py | 110 ++++++++++++++++++++++++------------ 8 files changed, 137 insertions(+), 163 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 0aa9e8d4..af9f4248 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -50,13 +50,6 @@ namespace python { -void readPipeline(PipelineExecutor* executor, std::string json) -{ - std::stringstream strm(json); - executor->getManager().readPipeline(strm); -} - - void addArrayReaders(PipelineExecutor* executor, std::vector> arrays) { // Make the symbols in pdal_base global so that they're accessible diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index aec9ef75..dd4aa4ed 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -45,7 +45,6 @@ namespace python class Array; -void readPipeline(PipelineExecutor* executor, std::string json); void addArrayReaders(PipelineExecutor* executor, std::vector> arrays); PyArrayObject* viewToNumpyArray(PointViewPtr view); PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index c37f7c8b..85a45073 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -226,12 +226,9 @@ char *PythonPointTable::getPoint(PointId idx) // StreamableExecutor -StreamableExecutor::StreamableExecutor(const char *json, int chunkSize, int prefetch) : - m_json(json), m_table(chunkSize, prefetch), m_manager(chunkSize), - m_log(Log::makeLog("pdal_python", &m_logStream)) -{ - m_manager.setLog(m_log); -} +StreamableExecutor::StreamableExecutor(std::string const& json, point_count_t chunkSize, int prefetch) : + PipelineExecutor(json), m_table(chunkSize, prefetch) +{} StreamableExecutor::~StreamableExecutor() { @@ -258,51 +255,10 @@ PyArrayObject *StreamableExecutor::executeNext() m_thread->join(); Py_END_ALLOW_THREADS m_thread.reset(); + m_executed = true; } return arr; } -std::string StreamableExecutor::getMetadata() const -{ - return pdal::Utils::toJSON(m_manager.getMetadata().clone("metadata")); -} - -std::string StreamableExecutor::getPipeline() const -{ - std::stringstream strm; - pdal::PipelineWriter::writePipeline(m_manager.getStage(), strm); - return strm.str(); -} - -int StreamableExecutor::getLogLevel() const -{ - return static_cast(m_log->getLevel()); -} - -std::string StreamableExecutor::getLog() const -{ - return m_logStream.str(); -} - -// Returns the active log level. -int StreamableExecutor::setLogLevel(int level) -{ - if (level < 0 || level > 8) - return getLogLevel(); - - m_log->setLevel(static_cast(level)); - return level; -} - -bool StreamableExecutor::validate() -{ - std::stringstream strm; - strm << m_json; - m_manager.readPipeline(strm); - m_manager.prepare(); - - return true; -} - } // namespace python } // namespace pdal diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index 06a94d25..8e913a0d 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -43,6 +43,7 @@ #include +#include #include #include @@ -83,28 +84,17 @@ class PythonPointTable : public StreamPointTable std::queue m_arrays; }; -class StreamableExecutor +class StreamableExecutor : public PipelineExecutor { public: - StreamableExecutor(const char *json, int chunkSize, int prefetch); + StreamableExecutor(std::string const& json, point_count_t chunkSize, int prefetch); ~StreamableExecutor(); - PyArrayObject *executeNext(); - std::string getMetadata() const; - std::string getPipeline() const; - std::string getSchema() const; - std::string getLog() const; - int getLogLevel() const; - int setLogLevel(int level); - bool validate(); + PyArrayObject* executeNext(); private: - std::string m_json; int m_prefetch; PythonPointTable m_table; - PipelineManager m_manager; - std::stringstream m_logStream; - LogPtr m_log; std::unique_ptr m_thread; }; diff --git a/pdal/__init__.py b/pdal/__init__.py index 74037412..09d58be9 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,18 +1,8 @@ __version__ = "2.4.2" -__all__ = [ - "Pipeline", - "PipelineIterator", - "Stage", - "Reader", - "Filter", - "Writer", - "dimensions", - "info", -] +__all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] from . import libpdalpython from .drivers import inject_pdal_drivers -from .libpdalpython import PipelineIterator from .pipeline import Filter, Pipeline, Reader, Stage, Writer inject_pdal_drivers() diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 12ad505b..fa607f03 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -94,6 +94,7 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": PipelineExecutor(string) except + const PipelineManager& getManagerConst() except + bool executed() except + + void read() except + int execute() except + string getPipeline() except + string getMetadata() except + @@ -102,63 +103,29 @@ cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": void setLogLevel(int) except + +cdef extern from "StreamableExecutor.hpp" namespace "pdal::python": + cdef cppclass StreamableExecutor(PipelineExecutor): + StreamableExecutor(string, int, int) except + + np.PyArrayObject* executeNext() except + + + cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: Array(np.PyArrayObject*) except + cdef extern from "PyPipeline.hpp" namespace "pdal::python": - void readPipeline(PipelineExecutor*, string) except + void addArrayReaders(PipelineExecutor*, vector[shared_ptr[Array]]) except + np.PyArrayObject* viewToNumpyArray(PointViewPtr) except + np.PyArrayObject* meshToNumpyArray(const TriangularMesh*) except + -cdef extern from "StreamableExecutor.hpp" namespace "pdal::python": - cdef cppclass StreamableExecutor: - StreamableExecutor(const char *, int, int) except + - np.PyArrayObject *executeNext() except + - bool validate() except + - string getMetadata() except + - string getPipeline() except + - string getSchema() except + - string getLog() except + - int getLogLevel() except + - int setLogLevel(int level) except + - - -cdef class PipelineIterator: - cdef int _chunk_size - cdef StreamableExecutor *_executor; - - def __cinit__(self, unicode json, list arrays=None, int chunk_size=10000, int prefetch=0): - self._chunk_size - self._executor = new StreamableExecutor(json.encode("UTF-8"), chunk_size, prefetch) -# self._add_arrays(arrays) - - property chunk_size: - def __get__(self): - return self._chunk_size - - def validate(self): - return self._executor.validate() - - def __iter__(self): - while True: - arrPtr = self._executor.executeNext() - if arrPtr is NULL: - break - arr = arrPtr - Py_DECREF(arr) - yield arr - cdef class Pipeline: - cdef unique_ptr[PipelineExecutor] _executor + cdef unique_ptr[StreamableExecutor] _executor cdef vector[shared_ptr[Array]] _inputs cdef int _loglevel - - def execute(self): - return self._get_executor().execute() + cdef int _chunk_size + cdef int _prefetch #========= writeable properties to be set before execution =========================== @@ -182,6 +149,26 @@ cdef class Pipeline: self._loglevel = value self._del_executor() + @property + def chunk_size(self): + return self._chunk_size + + @chunk_size.setter + def chunk_size(self, value): + assert value > 0 + self._chunk_size = value + self._del_executor() + + @property + def prefetch(self): + return self._prefetch + + @prefetch.setter + def prefetch(self, value): + assert value >= 0 + self._prefetch = value + self._del_executor() + #========= readable properties to be read after execution ============================ @property @@ -222,6 +209,21 @@ cdef class Pipeline: Py_DECREF(output[-1]) return output + #========= execution methods ========================================================= + + def execute(self): + return self._get_executor().execute() + + def __iter__(self): + cdef StreamableExecutor* executor = self._get_executor() + while True: + arr_ptr = executor.executeNext() + if arr_ptr is NULL: + break + arr = arr_ptr + Py_DECREF(arr) + yield arr + #========= non-public properties & methods =========================================== def _get_json(self): @@ -237,12 +239,12 @@ cdef class Pipeline: def _del_executor(self): self._executor.reset() - cdef PipelineExecutor* _get_executor(self) except NULL: + cdef StreamableExecutor* _get_executor(self) except NULL: if not self._executor: - json = self._get_json() - executor = new PipelineExecutor(json) + executor = new StreamableExecutor(self._get_json(), + self._chunk_size, self._prefetch) executor.setLogLevel(self._loglevel) - readPipeline(executor, json) + executor.read() addArrayReaders(executor, self._inputs) self._executor.reset(executor) return self._executor.get() diff --git a/pdal/pipeline.py b/pdal/pipeline.py index ef581e98..01800963 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -28,6 +28,8 @@ def __init__( spec: Union[None, str, Sequence[Stage]] = None, arrays: Sequence[np.ndarray] = (), loglevel: int = logging.ERROR, + chunk_size: int = 10000, + prefetch: int = 0, ): super().__init__() self._stages: List[Stage] = [] @@ -37,6 +39,8 @@ def __init__( self |= stage self.inputs = arrays self.loglevel = loglevel + self.chunk_size = chunk_size + self.prefetch = prefetch @property def stages(self) -> List[Stage]: diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 30dce928..3d11312e 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -12,18 +12,16 @@ DATADIRECTORY = os.path.join(os.path.dirname(__file__), "data") -def get_pipeline(filename): +def get_pipeline(filename, *, chunk_size=None, prefetch=None): with open(os.path.join(DATADIRECTORY, filename), "r") as f: if filename.endswith(".json"): pipeline = pdal.Pipeline(f.read()) elif filename.endswith(".py"): pipeline = eval(f.read(), vars(pdal)) - return pipeline - -def get_pipeline_iter(filename, chunk_size=10000): - with open(os.path.join(DATADIRECTORY, filename), "r") as f: - pipeline = pdal.PipelineIterator(f.read(), chunk_size=chunk_size, prefetch=3) - assert pipeline.validate() + if chunk_size is not None: + pipeline.chunk_size = chunk_size + if prefetch is not None: + pipeline.prefetch = prefetch return pipeline @@ -38,6 +36,11 @@ class TestPipeline: def test_construction(self, filename): """Can we construct a PDAL pipeline""" assert isinstance(get_pipeline(filename), pdal.Pipeline) + assert isinstance(get_pipeline(filename, chunk_size=100), pdal.Pipeline) + assert isinstance(get_pipeline(filename, prefetch=3), pdal.Pipeline) + assert isinstance( + get_pipeline(filename, chunk_size=100, prefetch=3), pdal.Pipeline + ) # construct Pipeline from a sequence of stages r = pdal.Reader("r") @@ -265,8 +268,6 @@ def test_streamable(self): assert (rs | fn | ws).streamable is False assert (rs | fs | wn).streamable is False - # fails against PDAL master; see https://github.com/PDAL/PDAL/issues/3566 - @pytest.mark.xfail @pytest.mark.parametrize("filename", ["reproject.json", "reproject.py"]) def test_logging(self, filename): """Can we fetch log output""" @@ -418,32 +419,71 @@ def test_meshio(self, filename): class TestPipelineIterator: - def test_construction(self): - """Can we construct a PDAL pipeline iterator""" - assert isinstance( - get_pipeline_iter("range.json"), pdal.PipelineIterator - ) - assert isinstance( - get_pipeline_iter("range.json", chunk_size=100), pdal.PipelineIterator - ) + @pytest.mark.xfail + def test_array(self): + """Can we fetch PDAL data as numpy arrays""" + ri = get_pipeline("range.json", chunk_size=100) + arrays = list(ri) + assert len(arrays) == 11 + concat_array = np.concatenate(arrays) + + r = get_pipeline("range.json") + count = r.execute() + arrays = r.arrays + assert len(arrays) == 1 + array = arrays[0] + assert count == len(array) - def test_validate(self): - """Do we complain with bad pipelines""" - bad_json = """ - [ - "nofile.las", - { - "type": "filters.range", - "limits": "Intensity[80:120)" - } - ] - """ - r = pdal.PipelineIterator(bad_json) + np.testing.assert_array_equal(array, concat_array) + + def test_metadata(self): + """Can we fetch PDAL metadata""" + ri = get_pipeline("range.json", chunk_size=100) with pytest.raises(RuntimeError): - r.validate() + ri.metadata + list(ri) + + r = get_pipeline("range.json") + with pytest.raises(RuntimeError): + r.metadata + r.execute() + + assert ri.metadata == r.metadata + + @pytest.mark.xfail + def test_schema(self): + """Fetching a schema works""" + ri = get_pipeline("range.json", chunk_size=100) + with pytest.raises(RuntimeError): + ri.schema + list(ri) + + r = get_pipeline("range.json") + with pytest.raises(RuntimeError): + r.schema + r.execute() + + assert ri.schema == r.schema + + @pytest.mark.skip("segfaults") + def test_merged_arrays(self): + """Can we load data from a list of arrays to PDAL""" + data = np.load(os.path.join(DATADIRECTORY, "test3d.npy")) + arrays = [data, data, data] + filter_intensity = """{ + "pipeline":[ + { + "type":"filters.range", + "limits":"Intensity[100:300)" + } + ] + }""" + arrays1 = list(pdal.Pipeline(filter_intensity, arrays, chunk_size=100)) + + p = pdal.Pipeline(filter_intensity, arrays) + p.execute() + arrays2 = p.arrays - def test_iterate(self): - pipeline = get_pipeline_iter("range.json") - print("Got into iterate!") - for arr in pipeline: - print(arr.shape) + assert len(arrays1) == len(arrays2) + for array1, array2 in zip(arrays1, arrays2): + np.testing.assert_array_equal(array1, array2) From a9c962b49fc9e04d66453afdf9e5dc6857f67df7 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 21 Oct 2021 21:17:21 +0300 Subject: [PATCH 055/161] Handle skips --- pdal/StreamableExecutor.cpp | 71 ++++++++++++++++++++++++------------- test/test_pipeline.py | 1 - 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 85a45073..20fc2e3f 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -108,6 +108,18 @@ void PythonPointTable::py_resizeArray(int np) PyArray_Dims dims{ sizes, 1 }; auto gil = PyGILState_Ensure(); + // copy the non-skipped elements to the beginning + npy_intp dest_idx = 0; + for (PointId src_idx = 0; src_idx < numPoints(); src_idx++) + if (!skip(src_idx)) + { + if (src_idx != dest_idx) + { + PyObject* src_item = PyArray_GETITEM(m_curArray, PyArray_GETPTR1(m_curArray, src_idx)); + PyArray_SETITEM(m_curArray, PyArray_GETPTR1(m_curArray, dest_idx), src_item); + } + dest_idx++; + } PyArray_Resize(m_curArray, &dims, true, NPY_CORDER); PyGILState_Release(gil); } @@ -124,30 +136,42 @@ PyObject *PythonPointTable::py_buildNumpyDescriptor() const // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} // - DimTypeList dims = layout()->dimTypes(); + auto dims = m_layout.dims(); + + // Need to sort the dimensions by offset + // Is there a better way? Can they be sorted by offset already? + auto sorter = [this](Dimension::Id id1, Dimension::Id id2) -> bool + { + return m_layout.dimOffset(id1) < m_layout.dimOffset(id2); + }; + std::sort(dims.begin(), dims.end(), sorter); + PyObject* names = PyList_New(dims.size()); PyObject* formats = PyList_New(dims.size()); for (size_t i = 0; i < dims.size(); ++i) { - DimType& dt = dims[i]; - std::string name = m_layout.dimName(dt.m_id); - npy_intp stride = Dimension::size(dt.m_type); - + auto id = dims[i]; std::string kind; - Dimension::BaseType b = Dimension::base(dt.m_type); - if (b == Dimension::BaseType::Unsigned) - kind = "u"; - else if (b == Dimension::BaseType::Signed) - kind = "i"; - else if (b == Dimension::BaseType::Floating) - kind = "f"; - else - throw pdal_error("Unable to map kind '" + kind + - "' to PDAL dimension type"); - - std::string type = kind + std::to_string(stride); + switch (Dimension::base(m_layout.dimType(id))) + { + case Dimension::BaseType::Unsigned: + kind = 'u'; + break; + case Dimension::BaseType::Signed: + kind = 'i'; + break; + case Dimension::BaseType::Floating: + kind = 'f'; + break; + default: + throw pdal_error("Unable to map kind '" + kind + "' to PDAL dimension type"); + } + + auto name = m_layout.dimName(id); PyList_SetItem(names, i, PyUnicode_FromString(name.c_str())); - PyList_SetItem(formats, i, PyUnicode_FromString(type.c_str())); + + auto format = kind + std::to_string(m_layout.dimSize(id)); + PyList_SetItem(formats, i, PyUnicode_FromString(format.c_str())); } PyObject* dict = PyDict_New(); @@ -158,12 +182,11 @@ PyObject *PythonPointTable::py_buildNumpyDescriptor() const void PythonPointTable::reset() { - point_count_t np = numPoints(); + point_count_t np = 0; + for (PointId idx = 0; idx < numPoints(); idx++) + if (!skip(idx)) + np++; - // If this is the last chunk, the size might be less than what's expected, so - // resize the array to match its true size. - // ABELL - This isn't quite right. We want to know if there are any skips and deal with those - // but I'm leaving that for the moment. if (np && np != m_limit) py_resizeArray(np); @@ -179,7 +202,7 @@ void PythonPointTable::reset() m_curArray = nullptr; } - bool done = np < m_limit; + bool done = numPoints() < m_limit; // If we just pushed the last chunk, push a nullptr so that a reader knows. if (done) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 3d11312e..54410d92 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -419,7 +419,6 @@ def test_meshio(self, filename): class TestPipelineIterator: - @pytest.mark.xfail def test_array(self): """Can we fetch PDAL data as numpy arrays""" ri = get_pipeline("range.json", chunk_size=100) From 799276ad9f7226b8367a2cfe86b26c64c34a5313 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Fri, 22 Oct 2021 22:55:02 +0300 Subject: [PATCH 056/161] Fix determining when the execution is really done in the presence of multiple readers --- pdal/StreamableExecutor.cpp | 22 ++++++++-------------- pdal/StreamableExecutor.hpp | 1 + test/test_pipeline.py | 14 ++++++-------- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 20fc2e3f..016f7dce 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -199,25 +199,18 @@ void PythonPointTable::reset() if (np) { m_arrays.push(m_curArray); - m_curArray = nullptr; + m_curArray = py_createArray(); + m_producedCv.notify_one(); } - - bool done = numPoints() < m_limit; - - // If we just pushed the last chunk, push a nullptr so that a reader knows. - if (done) - m_arrays.push(nullptr); - m_producedCv.notify_one(); - - if (done) - return; - while (m_arrays.size() > m_prefetch) m_consumedCv.wait(l); } +} - // Make a new array for data. - m_curArray = py_createArray(); +void PythonPointTable::done() +{ + m_arrays.push(nullptr); + m_producedCv.notify_one(); } PyArrayObject *PythonPointTable::fetchArray() @@ -267,6 +260,7 @@ PyArrayObject *StreamableExecutor::executeNext() m_thread.reset(new std::thread([this]() { m_manager.executeStream(m_table); + m_table.done(); })); } diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index 8e913a0d..2dd787b6 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -59,6 +59,7 @@ class PythonPointTable : public StreamPointTable ~PythonPointTable(); virtual void finalize(); + void done(); PyArrayObject *fetchArray(); protected: diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 54410d92..2e177a34 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -464,7 +464,6 @@ def test_schema(self): assert ri.schema == r.schema - @pytest.mark.skip("segfaults") def test_merged_arrays(self): """Can we load data from a list of arrays to PDAL""" data = np.load(os.path.join(DATADIRECTORY, "test3d.npy")) @@ -477,12 +476,11 @@ def test_merged_arrays(self): } ] }""" - arrays1 = list(pdal.Pipeline(filter_intensity, arrays, chunk_size=100)) - p = pdal.Pipeline(filter_intensity, arrays) p.execute() - arrays2 = p.arrays - - assert len(arrays1) == len(arrays2) - for array1, array2 in zip(arrays1, arrays2): - np.testing.assert_array_equal(array1, array2) + non_streaming_array = np.concatenate(p.arrays) + for chunk_size in range(5, 100, 5): + streaming_arrays = list(pdal.Pipeline(filter_intensity, arrays, + chunk_size=chunk_size)) + np.testing.assert_array_equal(np.concatenate(streaming_arrays), + non_streaming_array) From f0a7b65a880e6ac91d1c33dbc721e3b82696aa2a Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 23 Oct 2021 00:28:44 +0300 Subject: [PATCH 057/161] Refactor PythonPointTable --- pdal/StreamableExecutor.cpp | 54 ++++++++++++++----------------------- pdal/StreamableExecutor.hpp | 8 ++---- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 016f7dce..41cfc4f9 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -48,60 +48,46 @@ namespace python // PythonPointTable PythonPointTable::PythonPointTable(point_count_t limit, int prefetch) : - StreamPointTable(m_layout, limit), m_limit(limit), m_prefetch(prefetch), + StreamPointTable(m_layout, limit), m_prefetch(prefetch), m_curArray(nullptr), m_dtype(nullptr) {} PythonPointTable::~PythonPointTable() -{ - py_destroy(); -} - -void PythonPointTable::finalize() -{ - BasePointTable::finalize(); - py_createDescriptor(); - m_curArray = py_createArray(); -} - -void PythonPointTable::py_destroy() { auto gil = PyGILState_Ensure(); - Py_XDECREF(m_dtype); Py_XDECREF(m_curArray); - PyGILState_Release(gil); } -PyArrayObject *PythonPointTable::py_createArray() const +void PythonPointTable::finalize() { - auto gil = PyGILState_Ensure(); - - npy_intp size = (npy_intp)m_limit; - Py_INCREF(m_dtype); - PyArrayObject *arr = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, m_dtype, - 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); - - PyGILState_Release(gil); - return arr; -} + BasePointTable::finalize(); -void PythonPointTable::py_createDescriptor() -{ + // create dtype auto gil = PyGILState_Ensure(); - if (_import_array() < 0) std::cerr << "Could not import array!\n"; PyObject *dtype_dict = py_buildNumpyDescriptor(); if (PyArray_DescrConverter(dtype_dict, &m_dtype) == NPY_FAIL) - m_dtype = nullptr; + throw pdal_error("Unable to create numpy dtype"); Py_XDECREF(dtype_dict); + PyGILState_Release(gil); + + py_createArray(); +} +void PythonPointTable::py_createArray() +{ + auto gil = PyGILState_Ensure(); + npy_intp size = capacity(); + Py_INCREF(m_dtype); + m_curArray = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, m_dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); PyGILState_Release(gil); } -void PythonPointTable::py_resizeArray(int np) +void PythonPointTable::py_resizeArray(point_count_t np) { npy_intp sizes[1]; sizes[0] = np; @@ -187,7 +173,7 @@ void PythonPointTable::reset() if (!skip(idx)) np++; - if (np && np != m_limit) + if (np && np != capacity()) py_resizeArray(np); // This will keep putting arrays on the list until done, whether or not the consumer @@ -199,7 +185,7 @@ void PythonPointTable::reset() if (np) { m_arrays.push(m_curArray); - m_curArray = py_createArray(); + py_createArray(); m_producedCv.notify_one(); } while (m_arrays.size() > m_prefetch) @@ -236,7 +222,7 @@ PyArrayObject *PythonPointTable::fetchArray() char *PythonPointTable::getPoint(PointId idx) { - return (char *)PyArray_GETPTR1(m_curArray, (npy_intp)idx); + return (char *)PyArray_GETPTR1(m_curArray, idx); } diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index 2dd787b6..2ecf7fb4 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -68,13 +68,10 @@ class PythonPointTable : public StreamPointTable private: // All functions starting with py_ call Python things that need the GIL locked. - void py_destroy(); - PyArrayObject *py_createArray() const; - void py_createDescriptor(); - void py_resizeArray(int np); + void py_createArray(); + void py_resizeArray(point_count_t np); PyObject *py_buildNumpyDescriptor() const; - point_count_t m_limit; int m_prefetch; PointLayout m_layout; PyArrayObject *m_curArray; @@ -94,7 +91,6 @@ class StreamableExecutor : public PipelineExecutor PyArrayObject* executeNext(); private: - int m_prefetch; PythonPointTable m_table; std::unique_ptr m_thread; }; From 2c1ff3c35c1b925bf2a06af88b080ae060df892d Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 23 Oct 2021 01:02:29 +0300 Subject: [PATCH 058/161] Merge buildNumpyDescription function and PythonPointTable::py_buildNumpyDescriptor --- pdal/PyPipeline.cpp | 53 +++++++++++++++++++------------- pdal/PyPipeline.hpp | 1 + pdal/StreamableExecutor.cpp | 61 ++----------------------------------- pdal/StreamableExecutor.hpp | 1 - 4 files changed, 36 insertions(+), 80 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index af9f4248..d1ce3743 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -105,7 +105,7 @@ void addArrayReaders(PipelineExecutor* executor, std::vectordims(); + + // Ensure that the dimensions are sorted by offset + // Is there a better way? Can they be sorted by offset already? + auto sortByOffset = [layout](Dimension::Id id1, Dimension::Id id2) -> bool + { + return layout->dimOffset(id1) < layout->dimOffset(id2); + }; + auto dims = layout->dims(); + std::sort(dims.begin(), dims.end(), sortByOffset); + PyObject* names = PyList_New(dims.size()); PyObject* formats = PyList_New(dims.size()); for (size_t i = 0; i < dims.size(); ++i) { Dimension::Id id = dims[i]; - std::string name = view->dimName(id); - npy_intp stride = view->dimSize(id); - - std::string kind; - Dimension::BaseType b = Dimension::base(view->dimType(id)); - if (b == Dimension::BaseType::Unsigned) - kind = "u"; - else if (b == Dimension::BaseType::Signed) - kind = "i"; - else if (b == Dimension::BaseType::Floating) - kind = "f"; - else - throw pdal_error("Unable to map kind '" + kind + - "' to PDAL dimension type"); - - std::stringstream oss; - oss << kind << stride; + auto name = layout->dimName(id); PyList_SetItem(names, i, PyUnicode_FromString(name.c_str())); - PyList_SetItem(formats, i, PyUnicode_FromString(oss.str().c_str())); + + std::stringstream format; + switch (Dimension::base(layout->dimType(id))) + { + case Dimension::BaseType::Unsigned: + format << 'u'; + break; + case Dimension::BaseType::Signed: + format << 'i'; + break; + case Dimension::BaseType::Floating: + format << 'f'; + break; + default: + throw pdal_error("Unable to map dimension '" + name + "' to Numpy"); + } + format << layout->dimSize(id); + PyList_SetItem(formats, i, PyUnicode_FromString(format.str().c_str())); + } PyObject* dtype_dict = PyDict_New(); PyDict_SetItemString(dtype_dict, "names", names); @@ -154,7 +165,7 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view) if (_import_array() < 0) throw pdal_error("Could not import numpy.core.multiarray."); - PyObject* dtype_dict = buildNumpyDescription(view); + PyObject* dtype_dict = buildNumpyDescriptor(view->layout()); PyArray_Descr *dtype = nullptr; if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) throw pdal_error("Unable to build numpy dtype"); diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index dd4aa4ed..0f0aff4b 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -45,6 +45,7 @@ namespace python class Array; +PyObject* buildNumpyDescriptor(PointLayoutPtr layout); void addArrayReaders(PipelineExecutor* executor, std::vector> arrays); PyArrayObject* viewToNumpyArray(PointViewPtr view); PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 41cfc4f9..c10395d3 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -32,6 +32,7 @@ * OF SUCH DAMAGE. ****************************************************************************/ +#include "PyPipeline.hpp" #include "StreamableExecutor.hpp" #include @@ -68,7 +69,7 @@ void PythonPointTable::finalize() auto gil = PyGILState_Ensure(); if (_import_array() < 0) std::cerr << "Could not import array!\n"; - PyObject *dtype_dict = py_buildNumpyDescriptor(); + PyObject *dtype_dict = buildNumpyDescriptor(&m_layout); if (PyArray_DescrConverter(dtype_dict, &m_dtype) == NPY_FAIL) throw pdal_error("Unable to create numpy dtype"); Py_XDECREF(dtype_dict); @@ -110,62 +111,6 @@ void PythonPointTable::py_resizeArray(point_count_t np) PyGILState_Release(gil); } -PyObject *PythonPointTable::py_buildNumpyDescriptor() const -{ - // Build up a numpy dtype dictionary - // - // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', - // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], - // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', - // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', - // 'Classification', 'ScanAngleRank', 'UserData', - // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} - // - - auto dims = m_layout.dims(); - - // Need to sort the dimensions by offset - // Is there a better way? Can they be sorted by offset already? - auto sorter = [this](Dimension::Id id1, Dimension::Id id2) -> bool - { - return m_layout.dimOffset(id1) < m_layout.dimOffset(id2); - }; - std::sort(dims.begin(), dims.end(), sorter); - - PyObject* names = PyList_New(dims.size()); - PyObject* formats = PyList_New(dims.size()); - for (size_t i = 0; i < dims.size(); ++i) - { - auto id = dims[i]; - std::string kind; - switch (Dimension::base(m_layout.dimType(id))) - { - case Dimension::BaseType::Unsigned: - kind = 'u'; - break; - case Dimension::BaseType::Signed: - kind = 'i'; - break; - case Dimension::BaseType::Floating: - kind = 'f'; - break; - default: - throw pdal_error("Unable to map kind '" + kind + "' to PDAL dimension type"); - } - - auto name = m_layout.dimName(id); - PyList_SetItem(names, i, PyUnicode_FromString(name.c_str())); - - auto format = kind + std::to_string(m_layout.dimSize(id)); - PyList_SetItem(formats, i, PyUnicode_FromString(format.c_str())); - } - - PyObject* dict = PyDict_New(); - PyDict_SetItemString(dict, "names", names); - PyDict_SetItemString(dict, "formats", formats); - return dict; -} - void PythonPointTable::reset() { point_count_t np = 0; @@ -231,7 +176,7 @@ char *PythonPointTable::getPoint(PointId idx) StreamableExecutor::StreamableExecutor(std::string const& json, point_count_t chunkSize, int prefetch) : PipelineExecutor(json), m_table(chunkSize, prefetch) {} - + StreamableExecutor::~StreamableExecutor() { //ABELL - Hmmm. diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index 2ecf7fb4..48ec7fff 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -70,7 +70,6 @@ class PythonPointTable : public StreamPointTable // All functions starting with py_ call Python things that need the GIL locked. void py_createArray(); void py_resizeArray(point_count_t np); - PyObject *py_buildNumpyDescriptor() const; int m_prefetch; PointLayout m_layout; From 27103a99b96b90f75d33f48f119abe48f825764a Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 23 Oct 2021 12:06:36 +0300 Subject: [PATCH 059/161] Handle premature loop termination --- pdal/StreamableExecutor.cpp | 33 +++++++++++++++++++++++++++------ pdal/StreamableExecutor.hpp | 4 ++++ pdal/libpdalpython.pyx | 18 +++++++++++------- test/test_pipeline.py | 12 ++++++++++++ 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index c10395d3..cedf9716 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -138,6 +138,13 @@ void PythonPointTable::reset() } } +void PythonPointTable::disable() +{ + // TODO: uncomment the next line when/if StreamPointTable.m_capacity + // changes from private to protected + // m_capacity = 0; +} + void PythonPointTable::done() { m_arrays.push(nullptr); @@ -198,14 +205,28 @@ PyArrayObject *StreamableExecutor::executeNext() // Blocks until something is ready. PyArrayObject *arr = m_table.fetchArray(); if (arr == nullptr) + done(); + return arr; +} + +void StreamableExecutor::stop() +{ + if (m_thread) { - Py_BEGIN_ALLOW_THREADS - m_thread->join(); - Py_END_ALLOW_THREADS - m_thread.reset(); - m_executed = true; + m_table.disable(); + while (PyArrayObject* arr = m_table.fetchArray()) + Py_XDECREF(arr); + done(); } - return arr; +} + +void StreamableExecutor::done() +{ + Py_BEGIN_ALLOW_THREADS + m_thread->join(); + Py_END_ALLOW_THREADS + m_thread.reset(); + m_executed = true; } } // namespace python diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index 48ec7fff..bc7ea13c 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -59,6 +59,7 @@ class PythonPointTable : public StreamPointTable ~PythonPointTable(); virtual void finalize(); + void disable(); void done(); PyArrayObject *fetchArray(); @@ -88,8 +89,11 @@ class StreamableExecutor : public PipelineExecutor ~StreamableExecutor(); PyArrayObject* executeNext(); + void stop(); private: + void done(); + PythonPointTable m_table; std::unique_ptr m_thread; }; diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index fa607f03..76a66997 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -107,6 +107,7 @@ cdef extern from "StreamableExecutor.hpp" namespace "pdal::python": cdef cppclass StreamableExecutor(PipelineExecutor): StreamableExecutor(string, int, int) except + np.PyArrayObject* executeNext() except + + void stop() except + cdef extern from "PyArray.hpp" namespace "pdal::python": @@ -216,13 +217,16 @@ cdef class Pipeline: def __iter__(self): cdef StreamableExecutor* executor = self._get_executor() - while True: - arr_ptr = executor.executeNext() - if arr_ptr is NULL: - break - arr = arr_ptr - Py_DECREF(arr) - yield arr + try: + while True: + arr_ptr = executor.executeNext() + if arr_ptr is NULL: + break + arr = arr_ptr + Py_DECREF(arr) + yield arr + finally: + executor.stop() #========= non-public properties & methods =========================================== diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 2e177a34..fb07f932 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -484,3 +484,15 @@ def test_merged_arrays(self): chunk_size=chunk_size)) np.testing.assert_array_equal(np.concatenate(streaming_arrays), non_streaming_array) + + def test_premature_exit(self): + """Can we stop iterating before all arrays are fetched""" + r = get_pipeline("range.json") + r.execute() + assert len(r.arrays) == 1 + array = r.arrays[0] + + ri = get_pipeline("range.json", chunk_size=100) + for array2 in ri: + np.testing.assert_array_equal(array2, array[:len(array2)]) + break From d8508ae1804e029be45c1fe7022efff0d17c809d Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 12:46:08 +0300 Subject: [PATCH 060/161] API change: replace Pipeline.__iter__ with iterator(chunksize, prefetch) and add new PipelineIterator class --- pdal/libpdalpython.pyx | 131 +++++++++++++++++++++++------------------ pdal/pipeline.py | 8 --- test/test_pipeline.py | 80 +++++++++++++------------ 3 files changed, 114 insertions(+), 105 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 76a66997..821623cf 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -1,8 +1,10 @@ # distutils: language = c++ # cython: c_string_type=unicode, c_string_encoding=utf8 +import json from types import SimpleNamespace +cimport cython from cython.operator cimport dereference as deref from cpython.ref cimport Py_DECREF from libcpp cimport bool @@ -121,12 +123,33 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": np.PyArrayObject* meshToNumpyArray(const TriangularMesh*) except + -cdef class Pipeline: - cdef unique_ptr[StreamableExecutor] _executor +@cython.internal +cdef class PipelineResultsMixin: + @property + def log(self): + return self._get_executor().getLog() + + @property + def schema(self): + return json.loads(self._get_executor().getSchema()) + + @property + def pipeline(self): + return self._get_executor().getPipeline() + + @property + def metadata(self): + return self._get_executor().getMetadata() + + cdef PipelineExecutor* _get_executor(self) except NULL: + raise NotImplementedError("Abstract method") + + + +cdef class Pipeline(PipelineResultsMixin): + cdef unique_ptr[PipelineExecutor] _executor cdef vector[shared_ptr[Array]] _inputs cdef int _loglevel - cdef int _chunk_size - cdef int _prefetch #========= writeable properties to be set before execution =========================== @@ -150,44 +173,8 @@ cdef class Pipeline: self._loglevel = value self._del_executor() - @property - def chunk_size(self): - return self._chunk_size - - @chunk_size.setter - def chunk_size(self, value): - assert value > 0 - self._chunk_size = value - self._del_executor() - - @property - def prefetch(self): - return self._prefetch - - @prefetch.setter - def prefetch(self, value): - assert value >= 0 - self._prefetch = value - self._del_executor() - #========= readable properties to be read after execution ============================ - @property - def log(self): - return self._get_executor().getLog() - - @property - def schema(self): - return self._get_executor().getSchema() - - @property - def pipeline(self): - return self._get_executor().getPipeline() - - @property - def metadata(self): - return self._get_executor().getMetadata() - @property def arrays(self): cdef PipelineExecutor* executor = self._get_executor() @@ -215,18 +202,14 @@ cdef class Pipeline: def execute(self): return self._get_executor().execute() - def __iter__(self): - cdef StreamableExecutor* executor = self._get_executor() - try: - while True: - arr_ptr = executor.executeNext() - if arr_ptr is NULL: - break - arr = arr_ptr - Py_DECREF(arr) - yield arr - finally: - executor.stop() + def iterator(self, int chunk_size=10000, int prefetch = 0): + cdef StreamableExecutor* executor = new StreamableExecutor( + self._get_json(), chunk_size, prefetch + ) + self._configure_executor(executor) + it = PipelineIterator() + it.set_executor(executor) + return it #========= non-public properties & methods =========================================== @@ -243,12 +226,44 @@ cdef class Pipeline: def _del_executor(self): self._executor.reset() - cdef StreamableExecutor* _get_executor(self) except NULL: + cdef PipelineExecutor* _get_executor(self) except NULL: if not self._executor: - executor = new StreamableExecutor(self._get_json(), - self._chunk_size, self._prefetch) - executor.setLogLevel(self._loglevel) - executor.read() - addArrayReaders(executor, self._inputs) + executor = new PipelineExecutor(self._get_json()) + self._configure_executor(executor) self._executor.reset(executor) return self._executor.get() + + cdef _configure_executor(self, PipelineExecutor* executor): + executor.setLogLevel(self._loglevel) + executor.read() + addArrayReaders(executor, self._inputs) + + +cdef class PipelineIterator(PipelineResultsMixin): + cdef unique_ptr[StreamableExecutor] _executor + + cdef set_executor(self, StreamableExecutor* executor): + self._executor.reset(executor) + + def __dealloc__(self): + self._executor.get().stop() + + def __iter__(self): + return self + + def __next__(self): + cdef StreamableExecutor* executor = self._executor.get() + if executor.executed(): + raise StopIteration + + arr_ptr = executor.executeNext() + if arr_ptr is NULL: + executor.stop() + raise StopIteration + + arr = arr_ptr + Py_DECREF(arr) + return arr + + cdef PipelineExecutor* _get_executor(self) except NULL: + return self._executor.get() diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 01800963..bb655d46 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -28,8 +28,6 @@ def __init__( spec: Union[None, str, Sequence[Stage]] = None, arrays: Sequence[np.ndarray] = (), loglevel: int = logging.ERROR, - chunk_size: int = 10000, - prefetch: int = 0, ): super().__init__() self._stages: List[Stage] = [] @@ -39,8 +37,6 @@ def __init__( self |= stage self.inputs = arrays self.loglevel = loglevel - self.chunk_size = chunk_size - self.prefetch = prefetch @property def stages(self) -> List[Stage]: @@ -50,10 +46,6 @@ def stages(self) -> List[Stage]: def streamable(self) -> bool: return all(stage.streamable for stage in self._stages) - @property - def schema(self) -> Any: - return json.loads(super().schema) - @property def loglevel(self) -> int: return LogLevelFromPDAL[super().loglevel] diff --git a/test/test_pipeline.py b/test/test_pipeline.py index fb07f932..71dedbf5 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -2,7 +2,6 @@ import logging import os import sys -from functools import partial import numpy as np import pytest @@ -12,16 +11,12 @@ DATADIRECTORY = os.path.join(os.path.dirname(__file__), "data") -def get_pipeline(filename, *, chunk_size=None, prefetch=None): +def get_pipeline(filename): with open(os.path.join(DATADIRECTORY, filename), "r") as f: if filename.endswith(".json"): pipeline = pdal.Pipeline(f.read()) elif filename.endswith(".py"): pipeline = eval(f.read(), vars(pdal)) - if chunk_size is not None: - pipeline.chunk_size = chunk_size - if prefetch is not None: - pipeline.prefetch = prefetch return pipeline @@ -36,11 +31,6 @@ class TestPipeline: def test_construction(self, filename): """Can we construct a PDAL pipeline""" assert isinstance(get_pipeline(filename), pdal.Pipeline) - assert isinstance(get_pipeline(filename, chunk_size=100), pdal.Pipeline) - assert isinstance(get_pipeline(filename, prefetch=3), pdal.Pipeline) - assert isinstance( - get_pipeline(filename, chunk_size=100, prefetch=3), pdal.Pipeline - ) # construct Pipeline from a sequence of stages r = pdal.Reader("r") @@ -421,11 +411,6 @@ class TestPipelineIterator: def test_array(self): """Can we fetch PDAL data as numpy arrays""" - ri = get_pipeline("range.json", chunk_size=100) - arrays = list(ri) - assert len(arrays) == 11 - concat_array = np.concatenate(arrays) - r = get_pipeline("range.json") count = r.execute() arrays = r.arrays @@ -433,36 +418,44 @@ def test_array(self): array = arrays[0] assert count == len(array) - np.testing.assert_array_equal(array, concat_array) + for _ in range(10): + arrays = list(r.iterator(chunk_size=100)) + assert len(arrays) == 11 + concat_array = np.concatenate(arrays) + np.testing.assert_array_equal(array, concat_array) + + def test_StopIteration(self): + """Is StopIteration raised when the iterator is exhausted""" + r = get_pipeline("range.json") + it = r.iterator(chunk_size=100) + for array in it: + assert isinstance(array, np.ndarray) + with pytest.raises(StopIteration): + next(it) + assert next(it, None) is None def test_metadata(self): """Can we fetch PDAL metadata""" - ri = get_pipeline("range.json", chunk_size=100) - with pytest.raises(RuntimeError): - ri.metadata - list(ri) - r = get_pipeline("range.json") - with pytest.raises(RuntimeError): - r.metadata r.execute() - assert ri.metadata == r.metadata + it = r.iterator(chunk_size=100) + for _ in it: + pass + + assert r.metadata == it.metadata @pytest.mark.xfail def test_schema(self): """Fetching a schema works""" - ri = get_pipeline("range.json", chunk_size=100) - with pytest.raises(RuntimeError): - ri.schema - list(ri) - r = get_pipeline("range.json") - with pytest.raises(RuntimeError): - r.schema r.execute() - assert ri.schema == r.schema + it = r.iterator(chunk_size=100) + for _ in it: + pass + + assert r.schema == it.schema def test_merged_arrays(self): """Can we load data from a list of arrays to PDAL""" @@ -480,8 +473,7 @@ def test_merged_arrays(self): p.execute() non_streaming_array = np.concatenate(p.arrays) for chunk_size in range(5, 100, 5): - streaming_arrays = list(pdal.Pipeline(filter_intensity, arrays, - chunk_size=chunk_size)) + streaming_arrays = list(p.iterator(chunk_size=chunk_size)) np.testing.assert_array_equal(np.concatenate(streaming_arrays), non_streaming_array) @@ -492,7 +484,17 @@ def test_premature_exit(self): assert len(r.arrays) == 1 array = r.arrays[0] - ri = get_pipeline("range.json", chunk_size=100) - for array2 in ri: - np.testing.assert_array_equal(array2, array[:len(array2)]) - break + for _ in range(10): + for array2 in r.iterator(chunk_size=100): + np.testing.assert_array_equal(array2, array[:len(array2)]) + break + + def test_multiple_iterators(self): + """Can we create multiple independent iterators""" + r = get_pipeline("range.json") + it1 = r.iterator(chunk_size=100) + it2 = r.iterator(chunk_size=100) + for a1, a2 in zip(it1, it2): + np.testing.assert_array_equal(a1, a2) + assert next(it1, None) is None + assert next(it2, None) is None From 945b16e409dcad8a4d524de4f3de54cc0cf014d8 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 16:49:06 +0300 Subject: [PATCH 061/161] Override StreamableExecutor::getSchema --- pdal/StreamableExecutor.cpp | 11 +++++++++++ pdal/StreamableExecutor.hpp | 1 + pdal/libpdalpython.pyx | 4 ++++ test/test_pipeline.py | 1 - 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index cedf9716..282d0630 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -229,5 +229,16 @@ void StreamableExecutor::done() m_executed = true; } +std::string StreamableExecutor::getSchema() const +{ + if (!m_executed) + throw pdal_error("Pipeline has not been executed!"); + + std::stringstream strm; + MetadataNode root = m_table.layout()->toMetadata().clone("schema"); + pdal::Utils::toJSON(root, strm); + return strm.str(); +} + } // namespace python } // namespace pdal diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index bc7ea13c..baa04eec 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -88,6 +88,7 @@ class StreamableExecutor : public PipelineExecutor StreamableExecutor(std::string const& json, point_count_t chunkSize, int prefetch); ~StreamableExecutor(); + std::string getSchema() const; PyArrayObject* executeNext(); void stop(); diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 821623cf..c5b8dab7 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -248,6 +248,10 @@ cdef class PipelineIterator(PipelineResultsMixin): def __dealloc__(self): self._executor.get().stop() + @property + def schema(self): + return json.loads(self._executor.get().getSchema()) + def __iter__(self): return self diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 71dedbf5..92fc29f4 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -445,7 +445,6 @@ def test_metadata(self): assert r.metadata == it.metadata - @pytest.mark.xfail def test_schema(self): """Fetching a schema works""" r = get_pipeline("range.json") From bda277890b3ab9d0126d3f161ff9709e926898da Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 18:05:10 +0300 Subject: [PATCH 062/161] Parametrize pipeline iterator tests by filename and add copy of range.json test pipeline as Python expression --- pdal/libpdalpython.pyx | 1 + test/data/range.py | 1 + test/test_pipeline.py | 30 ++++++++++++++++++------------ 3 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 test/data/range.py diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index c5b8dab7..0a54bcb2 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -239,6 +239,7 @@ cdef class Pipeline(PipelineResultsMixin): addArrayReaders(executor, self._inputs) +@cython.internal cdef class PipelineIterator(PipelineResultsMixin): cdef unique_ptr[StreamableExecutor] _executor diff --git a/test/data/range.py b/test/data/range.py new file mode 100644 index 00000000..03a8aa26 --- /dev/null +++ b/test/data/range.py @@ -0,0 +1 @@ +Reader("test/data/autzen-utm.las") | Filter.range(limits="Intensity[80:120)") diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 92fc29f4..0fe7a773 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -409,9 +409,10 @@ def test_meshio(self, filename): class TestPipelineIterator: - def test_array(self): + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_array(self, filename): """Can we fetch PDAL data as numpy arrays""" - r = get_pipeline("range.json") + r = get_pipeline(filename) count = r.execute() arrays = r.arrays assert len(arrays) == 1 @@ -424,9 +425,10 @@ def test_array(self): concat_array = np.concatenate(arrays) np.testing.assert_array_equal(array, concat_array) - def test_StopIteration(self): + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_StopIteration(self, filename): """Is StopIteration raised when the iterator is exhausted""" - r = get_pipeline("range.json") + r = get_pipeline(filename) it = r.iterator(chunk_size=100) for array in it: assert isinstance(array, np.ndarray) @@ -434,9 +436,10 @@ def test_StopIteration(self): next(it) assert next(it, None) is None - def test_metadata(self): + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_metadata(self, filename): """Can we fetch PDAL metadata""" - r = get_pipeline("range.json") + r = get_pipeline(filename) r.execute() it = r.iterator(chunk_size=100) @@ -445,9 +448,10 @@ def test_metadata(self): assert r.metadata == it.metadata - def test_schema(self): + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_schema(self, filename): """Fetching a schema works""" - r = get_pipeline("range.json") + r = get_pipeline(filename) r.execute() it = r.iterator(chunk_size=100) @@ -476,9 +480,10 @@ def test_merged_arrays(self): np.testing.assert_array_equal(np.concatenate(streaming_arrays), non_streaming_array) - def test_premature_exit(self): + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_premature_exit(self, filename): """Can we stop iterating before all arrays are fetched""" - r = get_pipeline("range.json") + r = get_pipeline(filename) r.execute() assert len(r.arrays) == 1 array = r.arrays[0] @@ -488,9 +493,10 @@ def test_premature_exit(self): np.testing.assert_array_equal(array2, array[:len(array2)]) break - def test_multiple_iterators(self): + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_multiple_iterators(self, filename): """Can we create multiple independent iterators""" - r = get_pipeline("range.json") + r = get_pipeline(filename) it1 = r.iterator(chunk_size=100) it2 = r.iterator(chunk_size=100) for a1, a2 in zip(it1, it2): From 83f02e3c3d2d956d57db3532422aa6007a9c2e74 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 18:27:17 +0300 Subject: [PATCH 063/161] Raise RuntimeError on attempting to iterate non streamable pipelines --- pdal/libpdalpython.pyx | 4 ++++ test/test_pipeline.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 0a54bcb2..99190013 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -89,6 +89,7 @@ cdef extern from "pdal/PointView.hpp" namespace "pdal": cdef extern from "pdal/PipelineManager.hpp" namespace "pdal": cdef cppclass PipelineManager: const PointViewSet& views() const + bool pipelineStreamable() const cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": @@ -207,6 +208,9 @@ cdef class Pipeline(PipelineResultsMixin): self._get_json(), chunk_size, prefetch ) self._configure_executor(executor) + if not executor.getManagerConst().pipelineStreamable(): + raise RuntimeError("Pipeline is not streamable") + it = PipelineIterator() it.set_executor(executor) return it diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 0fe7a773..713ee0ea 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -408,6 +408,12 @@ def test_meshio(self, filename): class TestPipelineIterator: + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_non_streamable(self, filename): + r = get_pipeline(filename) + with pytest.raises(RuntimeError) as info: + r.iterator() + assert str(info.value) == "Pipeline is not streamable" @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_array(self, filename): From 162fc84c2343883c76a62a1505cd6b8f651bcf1e Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 18:32:47 +0300 Subject: [PATCH 064/161] Test RuntimeError message --- pdal/libpdalpython.pyx | 4 ++-- test/test_pipeline.py | 31 +++++++++++++++++++------------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 99190013..6b6d4698 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -180,7 +180,7 @@ cdef class Pipeline(PipelineResultsMixin): def arrays(self): cdef PipelineExecutor* executor = self._get_executor() if not executor.executed(): - raise RuntimeError("call execute() before fetching arrays") + raise RuntimeError("Pipeline has not been executed!") output = [] for view in executor.getManagerConst().views(): output.append(viewToNumpyArray(view)) @@ -191,7 +191,7 @@ cdef class Pipeline(PipelineResultsMixin): def meshes(self): cdef PipelineExecutor* executor = self._get_executor() if not executor.executed(): - raise RuntimeError("call execute() before fetching the mesh") + raise RuntimeError("Pipeline has not been executed!") output = [] for view in executor.getManagerConst().views(): output.append(meshToNumpyArray(deref(view).mesh())) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 713ee0ea..6169ac51 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -66,8 +66,9 @@ def test_execution(self, filename): def test_validate(self, filename): """Do we complain with bad pipelines""" r = get_pipeline(filename) - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError) as info: r.execute() + assert "No such file or directory" in str(info.value) @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_array(self, filename): @@ -85,8 +86,9 @@ def test_array(self, filename): def test_metadata(self, filename): """Can we fetch PDAL metadata""" r = get_pipeline(filename) - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError) as info: r.metadata + assert "Pipeline has not been executed" in str(info.value) r.execute() j = json.loads(r.metadata) @@ -96,8 +98,9 @@ def test_metadata(self, filename): def test_schema(self, filename): """Fetching a schema works""" r = get_pipeline(filename) - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError) as info: r.schema + assert "Pipeline has not been executed" in str(info.value) r.execute() assert r.schema["schema"]["dimensions"][0]["name"] == "X" @@ -106,8 +109,9 @@ def test_schema(self, filename): def test_pipeline(self, filename): """Can we fetch PDAL pipeline string""" r = get_pipeline(filename) - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError) as info: r.pipeline + assert "Pipeline has not been executed" in str(info.value) r.execute() assert json.loads(r.pipeline) == { @@ -131,8 +135,9 @@ def test_pipeline(self, filename): def test_no_execute(self, filename): """Does fetching arrays without executing throw an exception""" r = get_pipeline(filename) - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError) as info: r.arrays + assert "Pipeline has not been executed" in str(info.value) @pytest.mark.parametrize("filename", ["chip.json", "chip.py"]) def test_merged_arrays(self, filename): @@ -193,9 +198,9 @@ def test_pipe_stage_errors(self): (r, f) | (f, w) pipeline = r | w - with pytest.raises(RuntimeError) as ctx: + with pytest.raises(RuntimeError) as info: pipeline.execute() - assert "Undefined stage 'f'" in str(ctx.value) + assert "Undefined stage 'f'" in str(info.value) def test_inputs(self): """Can we combine pipelines with inputs""" @@ -380,8 +385,9 @@ class TestMesh: def test_no_execute(self, filename): """Does fetching meshes without executing throw an exception""" r = get_pipeline(filename) - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError) as info: r.meshes + assert "Pipeline has not been executed" in str(info.value) @pytest.mark.parametrize("filename", ["mesh.json", "mesh.py"]) def test_mesh(self, filename): @@ -413,7 +419,7 @@ def test_non_streamable(self, filename): r = get_pipeline(filename) with pytest.raises(RuntimeError) as info: r.iterator() - assert str(info.value) == "Pipeline is not streamable" + assert "Pipeline is not streamable" in str(info.value) @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_array(self, filename): @@ -483,8 +489,9 @@ def test_merged_arrays(self): non_streaming_array = np.concatenate(p.arrays) for chunk_size in range(5, 100, 5): streaming_arrays = list(p.iterator(chunk_size=chunk_size)) - np.testing.assert_array_equal(np.concatenate(streaming_arrays), - non_streaming_array) + np.testing.assert_array_equal( + np.concatenate(streaming_arrays), non_streaming_array + ) @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_premature_exit(self, filename): @@ -496,7 +503,7 @@ def test_premature_exit(self, filename): for _ in range(10): for array2 in r.iterator(chunk_size=100): - np.testing.assert_array_equal(array2, array[:len(array2)]) + np.testing.assert_array_equal(array2, array[: len(array2)]) break @pytest.mark.parametrize("filename", ["range.json", "range.py"]) From 30b603f277bf9bd8d35a7506368ccc06e4a24989 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 18:57:08 +0300 Subject: [PATCH 065/161] Update README --- README.rst | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cbe9ab34..294c1b0b 100644 --- a/README.rst +++ b/README.rst @@ -187,6 +187,25 @@ PDAL and Python: ).pipeline(clamped) print(pipeline.execute()) # 387 points +Iterating Streamable Pipelines +................................................................................ +Streamable pipelines (pipelines that consist exclusively of streamable PDAL +stages) can be executed in streaming mode via ``Pipeline.iterator()``. This +returns an iterator object that yields Numpy arrays of up to ``chunk_size`` size +(default=10000) at a time. + +.. code-block:: python + + import pdal + pipeline = pdal.Reader("test/data/autzen-utm.las") | pdal.Filter.range(limits="Intensity[80:120)") + for array in pipeline.iterator(chunk_size=500): + print(len(array)) + # or to concatenate all arrays into one + # full_array = np.concatenate(list(pipeline)) + +``Pipeline.iterator()`` also takes an optional ``prefetch`` parameter (default=0) +to allow prefetching up to to this number of arrays in parallel and buffering +them until they are yielded to the caller. Accessing Mesh Data ................................................................................ @@ -236,17 +255,14 @@ USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into ti .. code-block:: python import pdal - import json import psycopg2 import io - pipe = [ - '.../python/test/data/1.2-with-color.las', - {"type": "filters.splitter", "length": 1000}, - {"type": "filters.delaunay"} - ] - - pl = pdal.Pipeline(json.dumps(pipe)) + pl = ( + pdal.Reader(".../python/test/data/1.2-with-color.las") + | pdal.Filter.splitter(length=1000) + | pdal.Filter.delaunay() + ) pl.execute() conn = psycopg(%CONNNECTION_STRING%) From cf00bce7d720adc5452c533eb80aade34024b06c Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 26 Oct 2021 19:40:02 +0300 Subject: [PATCH 066/161] Move StreamableExecutor.stop() to its destructor --- pdal/StreamableExecutor.cpp | 4 +--- pdal/libpdalpython.pyx | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 282d0630..95b74408 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -186,9 +186,7 @@ StreamableExecutor::StreamableExecutor(std::string const& json, point_count_t ch StreamableExecutor::~StreamableExecutor() { - //ABELL - Hmmm. - if (m_thread) - m_thread->join(); + stop(); } PyArrayObject *StreamableExecutor::executeNext() diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 6b6d4698..42c8be4f 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -250,9 +250,6 @@ cdef class PipelineIterator(PipelineResultsMixin): cdef set_executor(self, StreamableExecutor* executor): self._executor.reset(executor) - def __dealloc__(self): - self._executor.get().stop() - @property def schema(self): return json.loads(self._executor.get().getSchema()) From 139e00a7a446cedc175a7c2fb09c06b34a3543c6 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 2 Nov 2021 12:02:56 +0200 Subject: [PATCH 067/161] Replace pdal::PipelineExecutor with a new pdal::python::PipelineExecutor --- pdal/PyPipeline.cpp | 83 ++++++++++++++++++++++++++++++++----- pdal/PyPipeline.hpp | 34 ++++++++++++--- pdal/StreamableExecutor.cpp | 25 +++++------ pdal/StreamableExecutor.hpp | 17 ++++---- pdal/libpdalpython.pyx | 59 +++++++------------------- 5 files changed, 135 insertions(+), 83 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index d1ce3743..921168cc 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -39,18 +39,82 @@ #include #endif -#include - -#include -#include - namespace pdal { namespace python { +PipelineExecutor::PipelineExecutor( + std::string const& json, std::vector> arrays, int level) +{ + if (level < 0 || level > 8) + throw pdal_error("log level must be between 0 and 8!"); + + LogPtr log(Log::makeLog("pypipeline", &m_logStream)); + log->setLevel(static_cast(level)); + m_manager.setLog(log); + + std::stringstream strm; + strm << json; + m_manager.readPipeline(strm); + + addArrayReaders(arrays); +} + + +point_count_t PipelineExecutor::execute() +{ + point_count_t count = m_manager.execute(); + m_executed = true; + return count; +} + + +const PointViewSet& PipelineExecutor::views() const +{ + if (!m_executed) + throw pdal_error("Pipeline has not been executed!"); + + return m_manager.views(); +} + + +std::string PipelineExecutor::getPipeline() const +{ + if (!m_executed) + throw pdal_error("Pipeline has not been executed!"); + + std::stringstream strm; + pdal::PipelineWriter::writePipeline(m_manager.getStage(), strm); + return strm.str(); +} + + +std::string PipelineExecutor::getMetadata() const +{ + if (!m_executed) + throw pdal_error("Pipeline has not been executed!"); + + std::stringstream strm; + MetadataNode root = m_manager.getMetadata().clone("metadata"); + pdal::Utils::toJSON(root, strm); + return strm.str(); +} + + +std::string PipelineExecutor::getSchema() const +{ + if (!m_executed) + throw pdal_error("Pipeline has not been executed!"); + + std::stringstream strm; + MetadataNode root = pointTable().layout()->toMetadata().clone("schema"); + pdal::Utils::toJSON(root, strm); + return strm.str(); +} + -void addArrayReaders(PipelineExecutor* executor, std::vector> arrays) +void PipelineExecutor::addArrayReaders(std::vector> arrays) { // Make the symbols in pdal_base global so that they're accessible // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, @@ -65,8 +129,7 @@ void addArrayReaders(PipelineExecutor* executor, std::vectorgetManager(); - std::vector roots = manager.roots(); + std::vector roots = m_manager.roots(); if (roots.size() != 1) throw pdal_error("Filter pipeline must contain a single root stage."); @@ -81,7 +144,7 @@ void addArrayReaders(PipelineExecutor* executor, std::vectorshape())); - Stage& s = manager.makeReader("", "readers.memoryview", options); + Stage& s = m_manager.makeReader("", "readers.memoryview", options); MemoryViewReader& r = dynamic_cast(s); for (auto f : array->fields()) r.pushField(f); @@ -101,7 +164,7 @@ void addArrayReaders(PipelineExecutor* executor, std::vectorsetInput(r); } - manager.validateStageOptions(); + m_manager.validateStageOptions(); } diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index 0f0aff4b..f7c4a81c 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -34,8 +34,7 @@ #pragma once -#include - +#include #include namespace pdal @@ -43,12 +42,37 @@ namespace pdal namespace python { -class Array; - PyObject* buildNumpyDescriptor(PointLayoutPtr layout); -void addArrayReaders(PipelineExecutor* executor, std::vector> arrays); PyArrayObject* viewToNumpyArray(PointViewPtr view); PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); +class Array; + +class PDAL_DLL PipelineExecutor { +public: + PipelineExecutor(std::string const& json, std::vector> arrays, int level); + virtual ~PipelineExecutor() = default; + + point_count_t execute(); + + bool executed() const { return m_executed; } + const PointViewSet& views() const; + std::string getPipeline() const; + std::string getMetadata() const; + std::string getSchema() const; + std::string getLog() const { return m_logStream.str(); } + +protected: + virtual ConstPointTableRef pointTable() const { return m_manager.pointTable(); } + + pdal::PipelineManager m_manager; + bool m_executed = false; + +private: + void addArrayReaders(std::vector> arrays); + + std::stringstream m_logStream; +}; + } // namespace python } // namespace pdal diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 95b74408..629161de 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -180,9 +180,17 @@ char *PythonPointTable::getPoint(PointId idx) // StreamableExecutor -StreamableExecutor::StreamableExecutor(std::string const& json, point_count_t chunkSize, int prefetch) : - PipelineExecutor(json), m_table(chunkSize, prefetch) -{} +StreamableExecutor::StreamableExecutor(std::string const& json, + std::vector> arrays, + int level, + point_count_t chunkSize, + int prefetch) + : PipelineExecutor(json, arrays, level) + , m_table(chunkSize, prefetch) +{ + if (!m_manager.pipelineStreamable()) + throw pdal_error("Pipeline is not streamable"); +} StreamableExecutor::~StreamableExecutor() { @@ -227,16 +235,5 @@ void StreamableExecutor::done() m_executed = true; } -std::string StreamableExecutor::getSchema() const -{ - if (!m_executed) - throw pdal_error("Pipeline has not been executed!"); - - std::stringstream strm; - MetadataNode root = m_table.layout()->toMetadata().clone("schema"); - pdal::Utils::toJSON(root, strm); - return strm.str(); -} - } // namespace python } // namespace pdal diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index baa04eec..bc2eef57 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -34,18 +34,10 @@ #pragma once -#include #include -#include -#include #include -#include -#include - -#include -#include -#include +#include "PyPipeline.hpp" namespace pdal { @@ -85,7 +77,11 @@ class PythonPointTable : public StreamPointTable class StreamableExecutor : public PipelineExecutor { public: - StreamableExecutor(std::string const& json, point_count_t chunkSize, int prefetch); + StreamableExecutor(std::string const& json, + std::vector> arrays, + int level, + point_count_t chunkSize, + int prefetch); ~StreamableExecutor(); std::string getSchema() const; @@ -94,6 +90,7 @@ class StreamableExecutor : public PipelineExecutor private: void done(); + ConstPointTableRef pointTable() const { return m_table; } PythonPointTable m_table; std::unique_ptr m_thread; diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 42c8be4f..e2c707b4 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -86,29 +86,24 @@ cdef extern from "pdal/PointView.hpp" namespace "pdal": ctypedef cpp_set[PointViewPtr] PointViewSet -cdef extern from "pdal/PipelineManager.hpp" namespace "pdal": - cdef cppclass PipelineManager: - const PointViewSet& views() const - bool pipelineStreamable() const - +cdef extern from "PyPipeline.hpp" namespace "pdal::python": + np.PyArrayObject* viewToNumpyArray(PointViewPtr) except + + np.PyArrayObject* meshToNumpyArray(const TriangularMesh*) except + -cdef extern from "pdal/PipelineExecutor.hpp" namespace "pdal": cdef cppclass PipelineExecutor: - PipelineExecutor(string) except + - const PipelineManager& getManagerConst() except + - bool executed() except + - void read() except + + PipelineExecutor(string, vector[shared_ptr[Array]], int) except + int execute() except + + bool executed() except + string getPipeline() except + string getMetadata() except + string getSchema() except + string getLog() except + - void setLogLevel(int) except + + const PointViewSet& views() except + cdef extern from "StreamableExecutor.hpp" namespace "pdal::python": cdef cppclass StreamableExecutor(PipelineExecutor): - StreamableExecutor(string, int, int) except + + StreamableExecutor(string, vector[shared_ptr[Array]], int, int, int) except + np.PyArrayObject* executeNext() except + void stop() except + @@ -118,12 +113,6 @@ cdef extern from "PyArray.hpp" namespace "pdal::python": Array(np.PyArrayObject*) except + -cdef extern from "PyPipeline.hpp" namespace "pdal::python": - void addArrayReaders(PipelineExecutor*, vector[shared_ptr[Array]]) except + - np.PyArrayObject* viewToNumpyArray(PointViewPtr) except + - np.PyArrayObject* meshToNumpyArray(const TriangularMesh*) except + - - @cython.internal cdef class PipelineResultsMixin: @property @@ -179,10 +168,8 @@ cdef class Pipeline(PipelineResultsMixin): @property def arrays(self): cdef PipelineExecutor* executor = self._get_executor() - if not executor.executed(): - raise RuntimeError("Pipeline has not been executed!") output = [] - for view in executor.getManagerConst().views(): + for view in executor.views(): output.append(viewToNumpyArray(view)) Py_DECREF(output[-1]) return output @@ -190,10 +177,8 @@ cdef class Pipeline(PipelineResultsMixin): @property def meshes(self): cdef PipelineExecutor* executor = self._get_executor() - if not executor.executed(): - raise RuntimeError("Pipeline has not been executed!") output = [] - for view in executor.getManagerConst().views(): + for view in executor.views(): output.append(meshToNumpyArray(deref(view).mesh())) Py_DECREF(output[-1]) return output @@ -204,15 +189,10 @@ cdef class Pipeline(PipelineResultsMixin): return self._get_executor().execute() def iterator(self, int chunk_size=10000, int prefetch = 0): - cdef StreamableExecutor* executor = new StreamableExecutor( - self._get_json(), chunk_size, prefetch - ) - self._configure_executor(executor) - if not executor.getManagerConst().pipelineStreamable(): - raise RuntimeError("Pipeline is not streamable") - it = PipelineIterator() - it.set_executor(executor) + it.set_executor(new StreamableExecutor( + self._get_json(), self._inputs, self._loglevel, chunk_size, prefetch + )) return it #========= non-public properties & methods =========================================== @@ -232,16 +212,11 @@ cdef class Pipeline(PipelineResultsMixin): cdef PipelineExecutor* _get_executor(self) except NULL: if not self._executor: - executor = new PipelineExecutor(self._get_json()) - self._configure_executor(executor) - self._executor.reset(executor) + self._executor.reset(new PipelineExecutor( + self._get_json(), self._inputs, self._loglevel) + ) return self._executor.get() - cdef _configure_executor(self, PipelineExecutor* executor): - executor.setLogLevel(self._loglevel) - executor.read() - addArrayReaders(executor, self._inputs) - @cython.internal cdef class PipelineIterator(PipelineResultsMixin): @@ -250,10 +225,6 @@ cdef class PipelineIterator(PipelineResultsMixin): cdef set_executor(self, StreamableExecutor* executor): self._executor.reset(executor) - @property - def schema(self): - return json.loads(self._executor.get().getSchema()) - def __iter__(self): return self From 277135f6efb7ab3ebec9515c80322a487989ae96 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 4 Nov 2021 01:11:21 +0200 Subject: [PATCH 068/161] Simplify StreamableExecutor --- pdal/PyPipeline.hpp | 1 - pdal/StreamableExecutor.cpp | 48 ++++++++++++++----------------------- pdal/StreamableExecutor.hpp | 3 --- pdal/libpdalpython.pyx | 16 ++++--------- 4 files changed, 22 insertions(+), 46 deletions(-) diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index f7c4a81c..95308070 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -55,7 +55,6 @@ class PDAL_DLL PipelineExecutor { point_count_t execute(); - bool executed() const { return m_executed; } const PointViewSet& views() const; std::string getPipeline() const; std::string getMetadata() const; diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 629161de..4964f4c4 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -190,49 +190,37 @@ StreamableExecutor::StreamableExecutor(std::string const& json, { if (!m_manager.pipelineStreamable()) throw pdal_error("Pipeline is not streamable"); -} - -StreamableExecutor::~StreamableExecutor() -{ - stop(); -} -PyArrayObject *StreamableExecutor::executeNext() -{ - if (!m_thread) + m_thread.reset(new std::thread([this]() { - m_thread.reset(new std::thread([this]() - { - m_manager.executeStream(m_table); - m_table.done(); - })); - } - - // Blocks until something is ready. - PyArrayObject *arr = m_table.fetchArray(); - if (arr == nullptr) - done(); - return arr; + m_manager.executeStream(m_table); + m_table.done(); + })); } -void StreamableExecutor::stop() +StreamableExecutor::~StreamableExecutor() { - if (m_thread) + if (!m_executed) { m_table.disable(); while (PyArrayObject* arr = m_table.fetchArray()) Py_XDECREF(arr); - done(); } -} - -void StreamableExecutor::done() -{ Py_BEGIN_ALLOW_THREADS m_thread->join(); Py_END_ALLOW_THREADS - m_thread.reset(); - m_executed = true; +} + +PyArrayObject *StreamableExecutor::executeNext() +{ + PyArrayObject* arr = nullptr; + if (!m_executed) + { + arr = m_table.fetchArray(); + if (arr == nullptr) + m_executed = true; + } + return arr; } } // namespace python diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index bc2eef57..7c736316 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -84,12 +84,9 @@ class StreamableExecutor : public PipelineExecutor int prefetch); ~StreamableExecutor(); - std::string getSchema() const; PyArrayObject* executeNext(); - void stop(); private: - void done(); ConstPointTableRef pointTable() const { return m_table; } PythonPointTable m_table; diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index e2c707b4..288b5190 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -7,7 +7,6 @@ from types import SimpleNamespace cimport cython from cython.operator cimport dereference as deref from cpython.ref cimport Py_DECREF -from libcpp cimport bool from libcpp.memory cimport make_shared, shared_ptr, unique_ptr from libcpp.set cimport set as cpp_set from libcpp.string cimport string @@ -93,7 +92,6 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef cppclass PipelineExecutor: PipelineExecutor(string, vector[shared_ptr[Array]], int) except + int execute() except + - bool executed() except + string getPipeline() except + string getMetadata() except + string getSchema() except + @@ -105,7 +103,6 @@ cdef extern from "StreamableExecutor.hpp" namespace "pdal::python": cdef cppclass StreamableExecutor(PipelineExecutor): StreamableExecutor(string, vector[shared_ptr[Array]], int, int, int) except + np.PyArrayObject* executeNext() except + - void stop() except + cdef extern from "PyArray.hpp" namespace "pdal::python": @@ -222,25 +219,20 @@ cdef class Pipeline(PipelineResultsMixin): cdef class PipelineIterator(PipelineResultsMixin): cdef unique_ptr[StreamableExecutor] _executor - cdef set_executor(self, StreamableExecutor* executor): - self._executor.reset(executor) - def __iter__(self): return self def __next__(self): - cdef StreamableExecutor* executor = self._executor.get() - if executor.executed(): - raise StopIteration - - arr_ptr = executor.executeNext() + arr_ptr = self._executor.get().executeNext() if arr_ptr is NULL: - executor.stop() raise StopIteration arr = arr_ptr Py_DECREF(arr) return arr + cdef set_executor(self, StreamableExecutor* executor): + self._executor.reset(executor) + cdef PipelineExecutor* _get_executor(self) except NULL: return self._executor.get() From 033c61eaf41bd89dbc69be931d7ced4b279dd586 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 4 Nov 2021 12:57:45 +0200 Subject: [PATCH 069/161] Add Pipeline.execute_streaming() method --- README.rst | 8 +++++++- pdal/PyPipeline.cpp | 18 ++++++++++++++++++ pdal/PyPipeline.hpp | 14 ++++++++++++++ pdal/libpdalpython.pyx | 4 ++++ test/test_pipeline.py | 18 +++++++++++++++++- 5 files changed, 60 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 294c1b0b..9d914ae2 100644 --- a/README.rst +++ b/README.rst @@ -187,7 +187,7 @@ PDAL and Python: ).pipeline(clamped) print(pipeline.execute()) # 387 points -Iterating Streamable Pipelines +Executing Streamable Pipelines ................................................................................ Streamable pipelines (pipelines that consist exclusively of streamable PDAL stages) can be executed in streaming mode via ``Pipeline.iterator()``. This @@ -207,6 +207,12 @@ returns an iterator object that yields Numpy arrays of up to ``chunk_size`` size to allow prefetching up to to this number of arrays in parallel and buffering them until they are yielded to the caller. +If you just want to execute a streamable pipeline in streaming mode and don't +need to access the data points (typically when the pipeline has Writer stage(s)), +you can use the ``Pipeline.execute_streaming(chunk_size)`` method instead. This +is functionally equivalent to ``sum(map(len, pipeline.iterator(chunk_size)))`` +but more efficient as it avoids allocating and filling any arrays in memory. + Accessing Mesh Data ................................................................................ diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 921168cc..8c5de3a9 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -44,6 +44,16 @@ namespace pdal namespace python { + +void CountPointTable::reset() +{ + for (PointId idx = 0; idx < numPoints(); idx++) + if (!skip(idx)) + m_count++; + FixedPointTable::reset(); +} + + PipelineExecutor::PipelineExecutor( std::string const& json, std::vector> arrays, int level) { @@ -70,6 +80,14 @@ point_count_t PipelineExecutor::execute() } +point_count_t PipelineExecutor::executeStream(point_count_t streamLimit) +{ + CountPointTable table(streamLimit); + m_manager.executeStream(table); + m_executed = true; + return table.count(); +} + const PointViewSet& PipelineExecutor::views() const { if (!m_executed) diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index 95308070..1f0b6643 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -54,6 +54,7 @@ class PDAL_DLL PipelineExecutor { virtual ~PipelineExecutor() = default; point_count_t execute(); + point_count_t executeStream(point_count_t streamLimit); const PointViewSet& views() const; std::string getPipeline() const; @@ -73,5 +74,18 @@ class PDAL_DLL PipelineExecutor { std::stringstream m_logStream; }; +class CountPointTable : public FixedPointTable +{ +public: + CountPointTable(point_count_t capacity) : FixedPointTable(capacity), m_count(0) {} + point_count_t count() const { return m_count; } + +protected: + virtual void reset(); + +private: + point_count_t m_count; +}; + } // namespace python } // namespace pdal diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 288b5190..c9036458 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -92,6 +92,7 @@ cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef cppclass PipelineExecutor: PipelineExecutor(string, vector[shared_ptr[Array]], int) except + int execute() except + + int executeStream(int) except + string getPipeline() except + string getMetadata() except + string getSchema() except + @@ -185,6 +186,9 @@ cdef class Pipeline(PipelineResultsMixin): def execute(self): return self._get_executor().execute() + def execute_streaming(self, int chunk_size=10000): + return self._get_executor().executeStream(chunk_size) + def iterator(self, int chunk_size=10000, int prefetch = 0): it = PipelineIterator() it.set_executor(new StreamableExecutor( diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 6169ac51..0e18209e 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -56,12 +56,28 @@ def test_invalid_json(self, pipeline): pdal.Pipeline(pipeline) @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) - def test_execution(self, filename): + def test_execute(self, filename): """Can we execute a PDAL pipeline""" r = get_pipeline(filename) r.execute() assert len(r.pipeline) > 200 + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_execute_streaming(self, filename): + r = get_pipeline(filename) + assert r.streamable + count = r.execute() + count2 = r.execute_streaming(chunk_size=100) + assert count == count2 + + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_execute_streaming_non_streamable(self, filename): + r = get_pipeline(filename) + assert not r.streamable + with pytest.raises(RuntimeError) as info: + r.execute_streaming() + assert "Attempting to use stream mode" in str(info.value) + @pytest.mark.parametrize("filename", ["bad.json", "bad.py"]) def test_validate(self, filename): """Do we complain with bad pipelines""" From 97682f7701d40df1e037b2b422915764e6021b89 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 4 Nov 2021 14:38:57 +0200 Subject: [PATCH 070/161] Fix memleak in PythonPointTable::py_resizeArray --- pdal/StreamableExecutor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index 4964f4c4..f8701e1b 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -104,6 +104,7 @@ void PythonPointTable::py_resizeArray(point_count_t np) { PyObject* src_item = PyArray_GETITEM(m_curArray, PyArray_GETPTR1(m_curArray, src_idx)); PyArray_SETITEM(m_curArray, PyArray_GETPTR1(m_curArray, dest_idx), src_item); + Py_XDECREF(src_item); } dest_idx++; } From 0264cb0cb81760c73303a37a537c9b3377fa369c Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Thu, 11 Nov 2021 09:43:31 +0200 Subject: [PATCH 071/161] Fix test_validate for windows --- test/test_pipeline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 0e18209e..fc9fe953 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -84,7 +84,10 @@ def test_validate(self, filename): r = get_pipeline(filename) with pytest.raises(RuntimeError) as info: r.execute() - assert "No such file or directory" in str(info.value) + if os.name == "nt": + assert "Unable to open stream for" in str(info.value) + else: + assert "No such file or directory" in str(info.value) @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_array(self, filename): From 08da7b951bedbd5e199d2dbec5fc9a4d5a12d925 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 13 Nov 2021 14:25:35 +0200 Subject: [PATCH 072/161] Catch any exception thrown by the StreamableExecutor thread and rethrow it from executeNext --- pdal/StreamableExecutor.cpp | 12 ++++++++---- pdal/StreamableExecutor.hpp | 1 + test/test_pipeline.py | 5 +++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index f8701e1b..d171bba9 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -188,13 +188,15 @@ StreamableExecutor::StreamableExecutor(std::string const& json, int prefetch) : PipelineExecutor(json, arrays, level) , m_table(chunkSize, prefetch) + , m_exc(nullptr) { - if (!m_manager.pipelineStreamable()) - throw pdal_error("Pipeline is not streamable"); - m_thread.reset(new std::thread([this]() { - m_manager.executeStream(m_table); + try { + m_manager.executeStream(m_table); + } catch (...) { + m_exc = std::current_exception(); + } m_table.done(); })); } @@ -220,6 +222,8 @@ PyArrayObject *StreamableExecutor::executeNext() arr = m_table.fetchArray(); if (arr == nullptr) m_executed = true; + if (m_exc) + std::rethrow_exception(m_exc); } return arr; } diff --git a/pdal/StreamableExecutor.hpp b/pdal/StreamableExecutor.hpp index 7c736316..629948f0 100644 --- a/pdal/StreamableExecutor.hpp +++ b/pdal/StreamableExecutor.hpp @@ -91,6 +91,7 @@ class StreamableExecutor : public PipelineExecutor PythonPointTable m_table; std::unique_ptr m_thread; + std::exception_ptr m_exc; }; } // namespace python diff --git a/test/test_pipeline.py b/test/test_pipeline.py index fc9fe953..dbac18e6 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -436,9 +436,10 @@ class TestPipelineIterator: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_non_streamable(self, filename): r = get_pipeline(filename) + assert not r.streamable with pytest.raises(RuntimeError) as info: - r.iterator() - assert "Pipeline is not streamable" in str(info.value) + next(r.iterator(chunk_size=100)) + assert "Attempting to use stream mode" in str(info.value) @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_array(self, filename): From ce36eefa209b4c236fa0d54760c15c9daad71080 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Sat, 13 Nov 2021 15:22:06 +0200 Subject: [PATCH 073/161] Acquire the GIL in StreamableExecutor destructor --- pdal/StreamableExecutor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pdal/StreamableExecutor.cpp b/pdal/StreamableExecutor.cpp index d171bba9..ef9950e2 100644 --- a/pdal/StreamableExecutor.cpp +++ b/pdal/StreamableExecutor.cpp @@ -206,8 +206,10 @@ StreamableExecutor::~StreamableExecutor() if (!m_executed) { m_table.disable(); + auto gil = PyGILState_Ensure(); while (PyArrayObject* arr = m_table.fetchArray()) Py_XDECREF(arr); + PyGILState_Release(gil); } Py_BEGIN_ALLOW_THREADS m_thread->join(); From 217b4383c992bc9ec377b3514772acce06d02988 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 15 Nov 2021 17:22:01 +0200 Subject: [PATCH 074/161] Remove pio.py (#101) --- pdal/pio.py | 145 ----------------------------------------------- test/test_pio.py | 57 ------------------- 2 files changed, 202 deletions(-) delete mode 100644 pdal/pio.py delete mode 100644 test/test_pio.py diff --git a/pdal/pio.py b/pdal/pio.py deleted file mode 100644 index fbbc15e0..00000000 --- a/pdal/pio.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -This module provides a python-syntax interface for constructing and executing pdal-python json -pipelines. The API is not explicitly defined but stage names are validated against the pdal executable's drivers when possible. - -To construct pipeline stages, access the driver name from this module. This will create -a callable function where driver parameters can be specified as keyword arguments. For example: - ->>> from pdal import pio ->>> las_reader = pio.readers.las(filename="test.las") - -To construct a pipeline, sum stages together. - ->>> pipeline = pio.readers.las(filename="test.las") + pio.writers.ply(filename="test.ply") - -To execute a pipeline and return results, call `execute`. - ->>> arr = pipeline.execute() # returns a numpy structured array - -To access the pipelines as a dict (which may be dumped to json), call `spec`. - ->>> json.dumps(pipeline.spec) - -""" - -import types -import json -import subprocess -from functools import partial -from collections import defaultdict -from itertools import chain -import copy -import warnings - -import pdal - -try: - PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout - PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) - _PDAL_VALIDATE = True -except: - PDAL_DRIVERS = [] - _PDAL_VALIDATE = False - -DEFAULT_STAGE_PARAMS = defaultdict(dict) -DEFAULT_STAGE_PARAMS.update({ -# TODO: add stage specific default configurations -}) - - -class StageSpec(object): - def __init__(self, prefix, **kwargs): - self.prefix = prefix - self.key = ".".join([self.prefix, kwargs.get("type", "")]) - self.spec = DEFAULT_STAGE_PARAMS[self.key].copy() - self.spec.update(kwargs) - self.spec["type"] = self.key - # NOTE: special case to support reading files without passing an explicit reader - if (self.prefix in ["readers", "writers"]) and kwargs.get("type") == "auto": - del self.spec["type"] - - @property - def pipeline(self): - """ - Promote this stage to a `pdal.pio.PipelineSpec` with one `pdal.pio.StageSpec` - and return it. - """ - output = PipelineSpec() - output.add_stage(self) - return output - - def __getattr__(self, name): - if _PDAL_VALIDATE and (name not in dir(self)): - raise AttributeError(f"'{self.prefix}.{name}' is an invalid or unsupported PDAL stage") - return partial(self.__class__, self.prefix, type=name) - - def __str__(self): - return json.dumps(self.spec, indent=4) - - def __add__(self, other): - return self.pipeline + other - - def __dir__(self): - extra_keys = [e["name"][len(self.key):] for e in PDAL_DRIVERS if e["name"].startswith(self.key)] + ["auto"] - return super().__dir__() + [e for e in extra_keys if len(e) > 0] - - def execute(self): - return self.pipeline.execute() - - -readers = StageSpec("readers") -filters = StageSpec("filters") -writers = StageSpec("writers") - - -class PipelineSpec(object): - stages = [] - - def __init__(self, other=None): - if other is not None: - self.stages = copy.copy(other.stages) - - @property - def spec(self): - """ - Return a `dict` containing the pdal pipeline suitable for dumping to json - """ - return { - "pipeline": [stage.spec for stage in self.stages] - } - - def add_stage(self, stage): - """ - Add a StageSpec to the end of this pipeline, and return the updated result. - """ - assert isinstance(stage, StageSpec), "Expected StageSpec" - - self.stages.append(stage) - return self - - def __str__(self): - return json.dumps(self.spec, indent=4) - - def __add__(self, stage_or_pipeline): - assert isinstance(stage_or_pipeline, (StageSpec, PipelineSpec)), "Expected StageSpec or PipelineSpec" - - output = self.__class__(self) - if isinstance(stage_or_pipeline, StageSpec): - output.add_stage(stage_or_pipeline) - elif isinstance(stage_or_pipeline, PipelineSpec): - for stage in stage_or_pipeline.stages: - output.add_stage(stage) - return output - - def execute(self): - """ - Shortcut to execute and return the results of the pipeline. - """ - # TODO: do some validation before calling execute - - # TODO: some exception/error handling around pdal - pipeline = pdal.Pipeline(json.dumps(self.spec)) - # pipeline.validate() # NOTE: disabling this because it causes segfaults in certain cases - pipeline.execute() - - return pipeline.arrays[0] # NOTE: are there situation where arrays has multiple elements? diff --git a/test/test_pio.py b/test/test_pio.py deleted file mode 100644 index bcde6bcb..00000000 --- a/test/test_pio.py +++ /dev/null @@ -1,57 +0,0 @@ -import unittest -import json - -from pdal import pio - -dummy_pipeline = """{ - "pipeline": [ - { - "type": "readers.ply", - "filename": "dummyinput.ply" - }, - { - "type": "filters.outlier", - "method": "statistical", - "mean_k": 16, - "multiplier": 1.0 - }, - { - "type": "filters.range", - "limits": "Classification![7:7]" - }, - { - "type": "filters.normal" - }, - { - "type": "writers.ply", - "storage_mode": "ascii", - "precision": 4, - "filename": "dummyoutput.ply", - "dims": "X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ" - } - ] -}""" - - - -class TestPIOBasics(unittest.TestCase): - def test_pipeline_construction(self): - pipeline = (pio.readers.ply(filename="dummyinput.ply") + - pio.filters.outlier(method="statistical", mean_k=16, multiplier=1.0) + - pio.filters.range(limits="Classification![7:7]") + - pio.filters.normal() + pio.writers.ply(storage_mode="ascii", precision=4, filename="dummyoutput.ply", - dims="X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ")) - - self.assertIsInstance(pipeline, pio.PipelineSpec) - self.assertEqual(len(list(pipeline.stages)), 5) - self.assertEqual(json.dumps(pipeline.spec, indent=2), dummy_pipeline) - - auto_reader = pio.readers.auto(filename="dummyinput.las") - auto_writer = pio.writers.auto(filename="dummyoutput.las") - - self.assertIn("filename", auto_reader.spec) - self.assertNotIn("type", auto_reader.spec) - self.assertIn("filename", auto_reader.spec) - self.assertNotIn("type", auto_writer.spec) - self.assertEqual(auto_reader.prefix, "readers") - self.assertEqual(auto_writer.prefix, "writers") From 6cd7016a52c04969f17a02544fa704690e565f60 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Wed, 3 Nov 2021 20:51:51 +0200 Subject: [PATCH 075/161] Add pybind11 implementation --- .github/workflows/build.yml | 10 +- CMakeLists.txt | 1 + pdal/CMakeLists.txt | 7 +- pdal/__init__.py | 8 +- pdal/libpybind11.cpp | 177 ++++++++++++++++++++++++++++++++++++ pdal/pipeline.py | 7 +- pyproject.toml | 2 +- 7 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 pdal/libpybind11.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4e877586..81c7eb63 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: - name: Dependencies shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest meshio + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pybind11 pdal pytest meshio - name: Install shell: bash -l {0} @@ -54,6 +54,8 @@ jobs: $SKPATH/pdal_filters_python_test $SKPATH/pdal_io_numpy_test py.test -v + export PDAL_PYTHON_PYBIND11=1 + py.test -v windows: name: ${{ matrix.os }} - ${{ matrix.python-version }} @@ -76,7 +78,7 @@ jobs: shell: cmd /C CALL "{0}" run: | call conda activate test - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal pytest meshio + conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pybind11 pdal pytest meshio - name: Install shell: cmd /C CALL "{0}" @@ -95,6 +97,8 @@ jobs: %SKPATH%\cmake-build\pdal_filters_python_test.exe %SKPATH%\cmake-build\pdal_io_numpy_test.exe py.test -v + set PDAL_PYTHON_PYBIND11=1 + py.test -v dist: name: Distribution @@ -117,7 +121,7 @@ jobs: - name: Dependencies shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pdal + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pybind11 pdal - name: sdist shell: bash -l {0} diff --git a/CMakeLists.txt b/CMakeLists.txt index c8a533f8..b1eb5c32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ find_package(PDAL 2.1 REQUIRED) if(SKBUILD) find_package(PythonExtensions REQUIRED) find_package(Cython REQUIRED) + find_package(pybind11 CONFIG REQUIRED) message(STATUS "The project is built using scikit-build") endif() diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt index 04ea5730..a64e354c 100644 --- a/pdal/CMakeLists.txt +++ b/pdal/CMakeLists.txt @@ -7,4 +7,9 @@ target_include_directories(${extension} PRIVATE . ${Python3_NumPy_INCLUDE_DIRS}) target_link_libraries(${extension} ${PDAL_LIBRARIES}) python_extension_module(${extension}) -install(TARGETS ${extension} LIBRARY DESTINATION "pdal") +set(pb11ext "libpybind11") +pybind11_add_module(${pb11ext} MODULE ${EXTENSION_SRC} libpybind11.cpp) +target_include_directories(${pb11ext} PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) +target_link_libraries(${pb11ext} PRIVATE ${PDAL_LIBRARIES}) + +install(TARGETS ${extension} ${pb11ext} LIBRARY DESTINATION "pdal") diff --git a/pdal/__init__.py b/pdal/__init__.py index 09d58be9..3d2de21d 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,7 +1,13 @@ __version__ = "2.4.2" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -from . import libpdalpython +import os + +if os.getenv("PDAL_PYTHON_PYBIND11"): + from . import libpybind11 as libpdalpython +else: + from . import libpdalpython + from .drivers import inject_pdal_drivers from .pipeline import Filter, Pipeline, Reader, Stage, Writer diff --git a/pdal/libpybind11.cpp b/pdal/libpybind11.cpp new file mode 100644 index 00000000..cba4c6c6 --- /dev/null +++ b/pdal/libpybind11.cpp @@ -0,0 +1,177 @@ +#include +#include +#include + +#include +#include + +#include "PyArray.hpp" +#include "PyDimension.hpp" +#include "PyPipeline.hpp" +#include "StreamableExecutor.hpp" + +namespace py = pybind11; + +namespace pdal { + using namespace py::literals; + + py::object getInfo() { + return py::module_::import("types").attr("SimpleNamespace")( + "version"_a = pdal::Config::versionString(), + "major"_a = pdal::Config::versionMajor(), + "minor"_a = pdal::Config::versionMinor(), + "patch"_a = pdal::Config::versionPatch(), + "debug"_a = pdal::Config::debugInformation(), + "sha1"_a = pdal::Config::sha1(), + "plugin"_a = pdal::Config::pluginInstallPath() + ); + }; + + std::vector getDimensions() { + py::object np = py::module_::import("numpy"); + py::object dtype = np.attr("dtype"); + std::vector dims; + for (const auto& dim: getValidDimensions()) + { + py::dict d( + "name"_a=dim.name, + "description"_a=dim.description, + "dtype"_a=dtype(dim.type + std::to_string(dim.size)) + ); + dims.push_back(std::move(d)); + } + return dims; + }; + + using pdal::python::PipelineExecutor; + using pdal::python::StreamableExecutor; + + class PipelineIterator : public StreamableExecutor { + public: + using StreamableExecutor::StreamableExecutor; + + py::object getSchema() { + return py::module_::import("json").attr("loads")(StreamableExecutor::getSchema()); + } + + py::array executeNext() { + PyArrayObject* arr(StreamableExecutor::executeNext()); + if (!arr) + throw py::stop_iteration(); + + return py::reinterpret_steal((PyObject*)arr); + } + + }; + + class Pipeline { + public: + point_count_t execute() { return getExecutor()->execute(); } + + point_count_t executeStream(point_count_t streamLimit) { + return getExecutor()->executeStream(streamLimit); + } + + std::unique_ptr iterator(int chunk_size, int prefetch) { + return std::unique_ptr(new PipelineIterator( + getJson(), _inputs, _loglevel, chunk_size, prefetch + )); + } + + void setInputs(std::vector ndarrays) { + _inputs.clear(); + for (const auto& ndarray: ndarrays) { + PyArrayObject* ndarray_ptr = (PyArrayObject*)ndarray.ptr(); + _inputs.push_back(std::make_shared(ndarray_ptr)); + } + delExecutor(); + } + + int getLoglevel() { return _loglevel; } + + void setLogLevel(int level) { _loglevel = level; delExecutor(); } + + std::string getLog() { return getExecutor()->getLog(); } + + std::string getPipeline() { return getExecutor()->getPipeline(); } + + std::string getMetadata() { return getExecutor()->getMetadata(); } + + py::object getSchema() { + return py::module_::import("json").attr("loads")(getExecutor()->getSchema()); + } + + std::vector getArrays() { + std::vector output; + for (const auto &view: getExecutor()->views()) { + PyArrayObject* arr(pdal::python::viewToNumpyArray(view)); + output.push_back(py::reinterpret_steal((PyObject*)arr)); + } + return output; + } + + std::vector getMeshes() { + std::vector output; + for (const auto &view: getExecutor()->views()) { + PyArrayObject* arr(pdal::python::meshToNumpyArray(view->mesh())); + output.push_back(py::reinterpret_steal((PyObject*)arr)); + } + return output; + } + + std::string getJson() const { + PYBIND11_OVERRIDE_PURE_NAME(std::string, Pipeline, "_get_json", getJson); + } + + bool hasInputs() { return !_inputs.empty(); } + + void copyInputs(const Pipeline& other) { _inputs = other._inputs; } + + void delExecutor() { _executor.reset(); } + + PipelineExecutor* getExecutor() { + if (!_executor) + _executor.reset(new PipelineExecutor(getJson(), _inputs, _loglevel)); + return _executor.get(); + } + + private: + std::unique_ptr _executor; + std::vector> _inputs; + int _loglevel; + }; + + PYBIND11_MODULE(libpybind11, m) + { + py::class_(m, "PipelineIterator") + .def("__iter__", [](PipelineIterator &it) -> PipelineIterator& { return it; }) + .def("__next__", &PipelineIterator::executeNext) + .def_property_readonly("log", &PipelineIterator::getLog) + .def_property_readonly("schema", &PipelineIterator::getSchema) + .def_property_readonly("pipeline", &PipelineIterator::getPipeline) + .def_property_readonly("metadata", &PipelineIterator::getMetadata); + + py::class_(m, "Pipeline") + .def(py::init<>()) + .def("execute", &Pipeline::execute) + .def("execute_streaming", &Pipeline::executeStream, "chunk_size"_a=10000) + .def("iterator", &Pipeline::iterator, "chunk_size"_a=10000, "prefetch"_a=0) + .def_property("inputs", nullptr, &Pipeline::setInputs) + .def_property("loglevel", &Pipeline::getLoglevel, &Pipeline::setLogLevel) + .def_property_readonly("log", &Pipeline::getLog) + .def_property_readonly("schema", &Pipeline::getSchema) + .def_property_readonly("pipeline", &Pipeline::getPipeline) + .def_property_readonly("metadata", &Pipeline::getMetadata) + .def_property_readonly("arrays", &Pipeline::getArrays) + .def_property_readonly("meshes", &Pipeline::getMeshes) + .def_property_readonly("_has_inputs", &Pipeline::hasInputs) + .def("_copy_inputs", &Pipeline::copyInputs) + .def("_get_json", &Pipeline::getJson) + .def("_del_executor", &Pipeline::delExecutor); + m.def("getInfo", &getInfo); + m.def("getDimensions", &getDimensions); + m.def("infer_reader_driver", &StageFactory::inferReaderDriver); + m.def("infer_writer_driver", &StageFactory::inferWriterDriver); + }; + +}; // namespace pdal diff --git a/pdal/pipeline.py b/pdal/pipeline.py index bb655d46..d7c8888e 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -2,6 +2,7 @@ import json import logging +import os from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np @@ -11,7 +12,11 @@ except ModuleNotFoundError: # pragma: no cover Mesh = None -from . import drivers, libpdalpython +from . import drivers +if os.getenv("PDAL_PYTHON_PYBIND11"): + from . import libpybind11 as libpdalpython +else: + from . import libpdalpython LogLevelToPDAL = { logging.ERROR: 0, diff --git a/pyproject.toml b/pyproject.toml index 59a777cf..49a850ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = [ "wheel", "scikit-build", "cmake", "ninja", "pytest", "cython", "packaging","numpy"] +requires = [ "wheel", "scikit-build", "cmake", "ninja", "pytest", "cython", "pybind11[global]", "packaging", "numpy"] From de43193f28ca7dd116d3c807ff979948921121ae Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 15 Nov 2021 15:28:03 +0200 Subject: [PATCH 076/161] Remove Cython implementation --- .github/workflows/build.yml | 14 +- .gitignore | 1 - CMakeLists.txt | 7 - README.rst | 2 +- pdal/CMakeLists.txt | 21 +- pdal/__init__.py | 8 +- pdal/{libpybind11.cpp => libpdalpython.cpp} | 2 +- pdal/libpdalpython.pyx | 242 -------------------- pdal/pipeline.py | 7 +- pyproject.toml | 2 +- 10 files changed, 17 insertions(+), 289 deletions(-) rename pdal/{libpybind11.cpp => libpdalpython.cpp} (99%) delete mode 100644 pdal/libpdalpython.pyx diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81c7eb63..8aed3bf4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: - name: Dependencies shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pybind11 pdal pytest meshio + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} pybind11 pdal pytest meshio - name: Install shell: bash -l {0} @@ -53,9 +53,7 @@ jobs: export PDAL_DRIVER_PATH=$SKPATH $SKPATH/pdal_filters_python_test $SKPATH/pdal_io_numpy_test - py.test -v - export PDAL_PYTHON_PYBIND11=1 - py.test -v + py.test -v test/ windows: name: ${{ matrix.os }} - ${{ matrix.python-version }} @@ -78,7 +76,7 @@ jobs: shell: cmd /C CALL "{0}" run: | call conda activate test - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pybind11 pdal pytest meshio + conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} pybind11 pdal pytest meshio - name: Install shell: cmd /C CALL "{0}" @@ -96,9 +94,7 @@ jobs: set PDAL_DRIVER_PATH=%SKPATH%\cmake-build %SKPATH%\cmake-build\pdal_filters_python_test.exe %SKPATH%\cmake-build\pdal_io_numpy_test.exe - py.test -v - set PDAL_PYTHON_PYBIND11=1 - py.test -v + py.test -v test/ dist: name: Distribution @@ -121,7 +117,7 @@ jobs: - name: Dependencies shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} cython pybind11 pdal + run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} pybind11 pdal - name: sdist shell: bash -l {0} diff --git a/.gitignore b/.gitignore index 35f9baa3..4cb02b15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -pdal/libpdalpython.cpp *.pyc _skbuild/* .vscode/* diff --git a/CMakeLists.txt b/CMakeLists.txt index b1eb5c32..f0c2ecdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,13 +16,6 @@ find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) # find PDAL. Require 2.1+ find_package(PDAL 2.1 REQUIRED) -if(SKBUILD) - find_package(PythonExtensions REQUIRED) - find_package(Cython REQUIRED) - find_package(pybind11 CONFIG REQUIRED) - message(STATUS "The project is built using scikit-build") -endif() - add_subdirectory(pdal) # Taken and adapted from PDAL's cmake macros.cmake diff --git a/README.rst b/README.rst index 9d914ae2..37ca51b9 100644 --- a/README.rst +++ b/README.rst @@ -302,7 +302,7 @@ Requirements * PDAL 2.2+ * Python >=3.6 -* Cython (eg :code:`pip install cython`) +* Pybind11 (eg :code:`pip install pybind11[global]`) * Numpy (eg :code:`pip install numpy`) * Packaging (eg :code:`pip install packaging`) * scikit-build (eg :code:`pip install scikit-build`) diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt index a64e354c..8886a465 100644 --- a/pdal/CMakeLists.txt +++ b/pdal/CMakeLists.txt @@ -1,15 +1,8 @@ -set(EXTENSION_SRC PyArray.cpp PyPipeline.cpp StreamableExecutor.cpp) - +find_package(pybind11 REQUIRED) set(extension "libpdalpython") -add_cython_target(${extension} "libpdalpython.pyx" CXX PY3) -add_library(${extension} MODULE ${EXTENSION_SRC} libpdalpython) -target_include_directories(${extension} PRIVATE . ${Python3_NumPy_INCLUDE_DIRS}) -target_link_libraries(${extension} ${PDAL_LIBRARIES}) -python_extension_module(${extension}) - -set(pb11ext "libpybind11") -pybind11_add_module(${pb11ext} MODULE ${EXTENSION_SRC} libpybind11.cpp) -target_include_directories(${pb11ext} PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) -target_link_libraries(${pb11ext} PRIVATE ${PDAL_LIBRARIES}) - -install(TARGETS ${extension} ${pb11ext} LIBRARY DESTINATION "pdal") +pybind11_add_module(${extension} MODULE + PyArray.cpp PyPipeline.cpp StreamableExecutor.cpp libpdalpython.cpp +) +target_include_directories(${extension} PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) +target_link_libraries(${extension} PRIVATE ${PDAL_LIBRARIES}) +install(TARGETS ${extension} LIBRARY DESTINATION "pdal") diff --git a/pdal/__init__.py b/pdal/__init__.py index 3d2de21d..09d58be9 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,13 +1,7 @@ __version__ = "2.4.2" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -import os - -if os.getenv("PDAL_PYTHON_PYBIND11"): - from . import libpybind11 as libpdalpython -else: - from . import libpdalpython - +from . import libpdalpython from .drivers import inject_pdal_drivers from .pipeline import Filter, Pipeline, Reader, Stage, Writer diff --git a/pdal/libpybind11.cpp b/pdal/libpdalpython.cpp similarity index 99% rename from pdal/libpybind11.cpp rename to pdal/libpdalpython.cpp index cba4c6c6..b3dfca90 100644 --- a/pdal/libpybind11.cpp +++ b/pdal/libpdalpython.cpp @@ -141,7 +141,7 @@ namespace pdal { int _loglevel; }; - PYBIND11_MODULE(libpybind11, m) + PYBIND11_MODULE(libpdalpython, m) { py::class_(m, "PipelineIterator") .def("__iter__", [](PipelineIterator &it) -> PipelineIterator& { return it; }) diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx deleted file mode 100644 index c9036458..00000000 --- a/pdal/libpdalpython.pyx +++ /dev/null @@ -1,242 +0,0 @@ -# distutils: language = c++ -# cython: c_string_type=unicode, c_string_encoding=utf8 - -import json -from types import SimpleNamespace - -cimport cython -from cython.operator cimport dereference as deref -from cpython.ref cimport Py_DECREF -from libcpp.memory cimport make_shared, shared_ptr, unique_ptr -from libcpp.set cimport set as cpp_set -from libcpp.string cimport string -from libcpp.vector cimport vector - -cimport numpy as np -np.import_array() - - -cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": - cdef int versionMajor() except + - cdef int versionMinor() except + - cdef int versionPatch() except + - cdef string sha1() except+ - cdef string debugInformation() except+ - cdef string pluginInstallPath() except+ - cdef string versionString() except+ - -def getInfo(): - return SimpleNamespace( - version=versionString(), - major=versionMajor(), - minor=versionMinor(), - patch=versionPatch(), - debug=debugInformation(), - sha1=sha1(), - plugin=pluginInstallPath(), - ) - - -cdef extern from "PyDimension.hpp": - ctypedef struct Dimension: - string name; - string description; - int size; - string type; -## string units; // Not defined by PDAL yet - cdef vector[Dimension] getValidDimensions() except + - -def getDimensions(): - output = [] - for dim in getValidDimensions(): - output.append({ - 'name': dim.name, - 'description': dim.description, - 'dtype': np.dtype(dim.type + str(dim.size)) - }) - return output - - -cdef extern from "pdal/StageFactory.hpp" namespace "pdal": - cpdef cppclass StageFactory: - @staticmethod - string inferReaderDriver(string) - @staticmethod - string inferWriterDriver(string) - - -def infer_reader_driver(driver): - return StageFactory.inferReaderDriver(driver) - -def infer_writer_driver(driver): - return StageFactory.inferWriterDriver(driver) - - -cdef extern from "pdal/Mesh.hpp" namespace "pdal": - cdef cppclass TriangularMesh: - pass - - -cdef extern from "pdal/PointView.hpp" namespace "pdal": - cdef cppclass PointView: - TriangularMesh* mesh() - - ctypedef shared_ptr[PointView] PointViewPtr - ctypedef cpp_set[PointViewPtr] PointViewSet - - -cdef extern from "PyPipeline.hpp" namespace "pdal::python": - np.PyArrayObject* viewToNumpyArray(PointViewPtr) except + - np.PyArrayObject* meshToNumpyArray(const TriangularMesh*) except + - - cdef cppclass PipelineExecutor: - PipelineExecutor(string, vector[shared_ptr[Array]], int) except + - int execute() except + - int executeStream(int) except + - string getPipeline() except + - string getMetadata() except + - string getSchema() except + - string getLog() except + - const PointViewSet& views() except + - - -cdef extern from "StreamableExecutor.hpp" namespace "pdal::python": - cdef cppclass StreamableExecutor(PipelineExecutor): - StreamableExecutor(string, vector[shared_ptr[Array]], int, int, int) except + - np.PyArrayObject* executeNext() except + - - -cdef extern from "PyArray.hpp" namespace "pdal::python": - cdef cppclass Array: - Array(np.PyArrayObject*) except + - - -@cython.internal -cdef class PipelineResultsMixin: - @property - def log(self): - return self._get_executor().getLog() - - @property - def schema(self): - return json.loads(self._get_executor().getSchema()) - - @property - def pipeline(self): - return self._get_executor().getPipeline() - - @property - def metadata(self): - return self._get_executor().getMetadata() - - cdef PipelineExecutor* _get_executor(self) except NULL: - raise NotImplementedError("Abstract method") - - - -cdef class Pipeline(PipelineResultsMixin): - cdef unique_ptr[PipelineExecutor] _executor - cdef vector[shared_ptr[Array]] _inputs - cdef int _loglevel - - #========= writeable properties to be set before execution =========================== - - @property - def inputs(self): - raise AttributeError("unreadable attribute") - - @inputs.setter - def inputs(self, ndarrays): - self._inputs.clear() - for ndarray in ndarrays: - self._inputs.push_back(make_shared[Array](ndarray)) - self._del_executor() - - @property - def loglevel(self): - return self._loglevel - - @loglevel.setter - def loglevel(self, value): - self._loglevel = value - self._del_executor() - - #========= readable properties to be read after execution ============================ - - @property - def arrays(self): - cdef PipelineExecutor* executor = self._get_executor() - output = [] - for view in executor.views(): - output.append(viewToNumpyArray(view)) - Py_DECREF(output[-1]) - return output - - @property - def meshes(self): - cdef PipelineExecutor* executor = self._get_executor() - output = [] - for view in executor.views(): - output.append(meshToNumpyArray(deref(view).mesh())) - Py_DECREF(output[-1]) - return output - - #========= execution methods ========================================================= - - def execute(self): - return self._get_executor().execute() - - def execute_streaming(self, int chunk_size=10000): - return self._get_executor().executeStream(chunk_size) - - def iterator(self, int chunk_size=10000, int prefetch = 0): - it = PipelineIterator() - it.set_executor(new StreamableExecutor( - self._get_json(), self._inputs, self._loglevel, chunk_size, prefetch - )) - return it - - #========= non-public properties & methods =========================================== - - def _get_json(self): - raise NotImplementedError("Abstract method") - - @property - def _has_inputs(self): - return not self._inputs.empty() - - def _copy_inputs(self, Pipeline other): - self._inputs = other._inputs - - def _del_executor(self): - self._executor.reset() - - cdef PipelineExecutor* _get_executor(self) except NULL: - if not self._executor: - self._executor.reset(new PipelineExecutor( - self._get_json(), self._inputs, self._loglevel) - ) - return self._executor.get() - - -@cython.internal -cdef class PipelineIterator(PipelineResultsMixin): - cdef unique_ptr[StreamableExecutor] _executor - - def __iter__(self): - return self - - def __next__(self): - arr_ptr = self._executor.get().executeNext() - if arr_ptr is NULL: - raise StopIteration - - arr = arr_ptr - Py_DECREF(arr) - return arr - - cdef set_executor(self, StreamableExecutor* executor): - self._executor.reset(executor) - - cdef PipelineExecutor* _get_executor(self) except NULL: - return self._executor.get() diff --git a/pdal/pipeline.py b/pdal/pipeline.py index d7c8888e..bb655d46 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -2,7 +2,6 @@ import json import logging -import os from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np @@ -12,11 +11,7 @@ except ModuleNotFoundError: # pragma: no cover Mesh = None -from . import drivers -if os.getenv("PDAL_PYTHON_PYBIND11"): - from . import libpybind11 as libpdalpython -else: - from . import libpdalpython +from . import drivers, libpdalpython LogLevelToPDAL = { logging.ERROR: 0, diff --git a/pyproject.toml b/pyproject.toml index 49a850ca..82955c60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = [ "wheel", "scikit-build", "cmake", "ninja", "pytest", "cython", "pybind11[global]", "packaging", "numpy"] +requires = [ "wheel", "scikit-build", "cmake", "ninja", "pytest", "pybind11[global]", "packaging", "numpy"] From ff2dae0eb0640b724c440410d95fc1d2e48e00c8 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 16 Nov 2021 03:58:42 +0200 Subject: [PATCH 077/161] Remove duplicated & unused test data files --- pdal/test/PythonFilterTest.cpp | 6 +++--- pdal/test/data/from-module.json | 11 ----------- pdal/test/data/predicate-embed.json | 11 ----------- .../data/predicate-keep-ground-and-unclass.json | 12 ------------ pdal/test/data/predicate-keep-last-return.json | 15 --------------- .../data/predicate-keep-specified-returns.json | 12 ------------ pdal/test/data/programmable-update-y-dims.json | 11 ----------- test/__init__.py | 0 test/data/perlin.npy | Bin 80080 -> 0 bytes 9 files changed, 3 insertions(+), 75 deletions(-) delete mode 100644 pdal/test/data/from-module.json delete mode 100644 pdal/test/data/predicate-embed.json delete mode 100644 pdal/test/data/predicate-keep-ground-and-unclass.json delete mode 100644 pdal/test/data/predicate-keep-last-return.json delete mode 100644 pdal/test/data/predicate-keep-specified-returns.json delete mode 100644 pdal/test/data/programmable-update-y-dims.json delete mode 100644 test/__init__.py delete mode 100644 test/data/perlin.npy diff --git a/pdal/test/PythonFilterTest.cpp b/pdal/test/PythonFilterTest.cpp index d114ec93..8871eac0 100644 --- a/pdal/test/PythonFilterTest.cpp +++ b/pdal/test/PythonFilterTest.cpp @@ -139,7 +139,7 @@ TEST_F(PythonFilterTest, pipelineJSON) PipelineManager manager; manager.readPipeline( - Support::datapath("programmable-update-y-dims.json")); + Support::datapath("pipeline/programmable-update-y-dims.json")); manager.execute(); PointViewSet viewSet = manager.views(); EXPECT_EQ(viewSet.size(), 1u); @@ -571,7 +571,7 @@ TEST_F(PredicateFilterTest, PredicateFilterTest_PipelineJSON) { PipelineManager mgr; - mgr.readPipeline(Support::datapath("from-module.json")); + mgr.readPipeline(Support::datapath("pipeline/from-module.json")); point_count_t cnt = mgr.execute(); EXPECT_EQ(cnt, 1u); } @@ -580,7 +580,7 @@ TEST_F(PredicateFilterTest, PredicateFilterTest_EmbedJSON) { PipelineManager mgr; - mgr.readPipeline(Support::datapath("predicate-embed.json")); + mgr.readPipeline(Support::datapath("pipeline/predicate-embed.json")); point_count_t cnt = mgr.execute(); EXPECT_EQ(cnt, 1u); } diff --git a/pdal/test/data/from-module.json b/pdal/test/data/from-module.json deleted file mode 100644 index 1bcce4e8..00000000 --- a/pdal/test/data/from-module.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "pipeline":[ - "./pdal/test/data/1.2-with-color.las", - { - "type": "filters.python", - "script": "./pdal/test/data/test1.py", - "module": "anything", - "function": "fff" - } - ] -} diff --git a/pdal/test/data/predicate-embed.json b/pdal/test/data/predicate-embed.json deleted file mode 100644 index 1abc6911..00000000 --- a/pdal/test/data/predicate-embed.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "pipeline":[ - "./pdal/test/data/1.2-with-color.las", - { - "type":"filters.python", - "module":"anything", - "function":"fff", - "source":"import numpy as np\ndef fff(ins,outs):\n\tX = ins['X']\n\tResult = np.equal(X, 637501.67)\n\t#print X\n\t#print Mask\n\touts['Mask'] = Result\n\treturn True\n" - } - ] -} diff --git a/pdal/test/data/predicate-keep-ground-and-unclass.json b/pdal/test/data/predicate-keep-ground-and-unclass.json deleted file mode 100644 index 83666f2a..00000000 --- a/pdal/test/data/predicate-keep-ground-and-unclass.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "pipeline":[ - "./autzen.las", - { - "type":"filters.predicate", - "module":"anything", - "function":"filter", - "source":"import numpy as np\n\ndef filter(ins,outs):\n\tcls = ins['Classification']\n\n\tkeep_classes = [1,2]\n\n\t# Use the first test for our base array.\n\tkeep = np.equal(cls, keep_classes[0])\n\n\t# For 1:n, test each predicate and join back\n\t# to our existing predicate array\n\tfor k in range(1,len(keep_classes)):\n\t\tt = np.equal(cls, keep_classes[k])\n\t\tkeep = keep + t\n\n\touts['Mask'] = keep\n\treturn True" - }, - "out2.las" - ] -} diff --git a/pdal/test/data/predicate-keep-last-return.json b/pdal/test/data/predicate-keep-last-return.json deleted file mode 100644 index 434cfde6..00000000 --- a/pdal/test/data/predicate-keep-last-return.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "pipeline":[ - "./autzen.las", - { - "type":"filters.stats" - }, - { - "type":"filters.predicate", - "module":"anything", - "function":"filter", - "source":"import numpy as np\n\ndef filter(ins,outs):\n\tret = ins['ReturnNumber']\n\tret_no = ins['NumberOfReturns']\n\n\t# Use the first test for our base array.\n\trets = np.equal(ret, ret_no)\n\n\touts['Mask'] = rets\n\treturn True" - }, - "out2.las" - ] -} diff --git a/pdal/test/data/predicate-keep-specified-returns.json b/pdal/test/data/predicate-keep-specified-returns.json deleted file mode 100644 index f2052cfa..00000000 --- a/pdal/test/data/predicate-keep-specified-returns.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "pipeline":[ - "./autzen.las", - { - "type":"filters.predicate", - "module":"anything", - "function":"filter", - "source":"import numpy as np\n\ndef filter(ins,outs):\n\tret = ins['ReturnNumber']\n\n\tkeep_ret = [0, 1,2]\n\n\t# Use the first test for our base array.\n\tkeep = np.equal(ret, keep_ret[0])\n\n\t# For 1:n, test each predicate and join back\n\t# to our existing predicate array\n\tfor k in range(1, len(keep_ret)):\n\t\tt = np.equal(ret, keep_ret[k])\n\t\tkeep = keep + t\n\n\touts['Mask'] = keep\n\treturn True" - }, - "out2.las" - ] -} diff --git a/pdal/test/data/programmable-update-y-dims.json b/pdal/test/data/programmable-update-y-dims.json deleted file mode 100644 index 00b1731a..00000000 --- a/pdal/test/data/programmable-update-y-dims.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "pipeline":[ - "./pdal/test/data/autzen-utm.las", - { - "type":"filters.python", - "function":"myfunc", - "module":"derive", - "source":"import numpy as np\ndef myfunc(ins,outs):\n\tX = ins['Y']\n\tX1 = np.zeros(X.size, dtype=type(X[0])) + 314\n\touts['Y'] = X1\n\treturn True" - } - ] -} diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/data/perlin.npy b/test/data/perlin.npy deleted file mode 100644 index 457a1356be488c4ec781023f1150eae5f23f3743..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80080 zcmdSBc{Eku8$T|UQj#=DDwRr+Bn_k<6`~|X5|U^@DrwRrDUt?bLdeW5WS)7rri<&k z_ZmZ}G^;ciD)KuyTko|#YyH;u-|zSSQENNv1m`IdeeL!-nv*6t8#&dH?JjL88w|p_Y>e^YqyJgT@)MT_q*56JIF$jo6Msh zy^1mGa8#RtNG7ib32@;SJ)Yn}k1yoY z;|*l|>3&IUI8VYPU>0MluxwPxs!=qd@{iJkD=0Xk5J*Fj;oy*3` zR+Vv!s#&O{)T3MdyBN6{jY&Ip=i~JF R=Qc!)xO8@uGVd!U7xO&=D4TuissY^Kr z!8eU)4GNAXcnsD<~26Uf_5pFrj00WIMrh0h6{___bNnX!V&o(&b{df z!W)mik!mW2fiPqEW6Xkm`|mEi8O?@N&oeC|%ekNu{Ig!liwD^eCwu}-`0!CFB_7W6 zVfnQ(=kaYk80!gzV$ zFz4c@z!2Z(r?_~wUA7Vf=OXh*M2?O3dxb;eIHv^um65boTodGS^jm_P* zGC*@%>IEsS|JozhNBWQ4KO%2L{)oO2{Ui45B>SBh{a#{`b6?UJ!Ax`&7$^F0QxU$? zYo98wTZW9C4mWogR$|C9W#iJB)o9b7f8_8H4n|;^b?OohwvFoZHZf;oPm}+sqf0CB zN92s2ZN()hdaKjPbg%#yG(Ddn?2?Kf{E|v{*P0YIx2f zB?XkP4eN(>$A;G_kn;>0{bjy=rsjRtp_iI$zieol{?qtpbT$|_jv3dJkPT0S4xZ?H zoDDwh-1O1G*^q1KCf)6v4L4`rv_E8$4XT`z4>(J+A=2@F`i`$zuy(ZTQQy=oV2pop zF!yv8ye$ebowYg(LS-Im4gSgmS;dfs#xf>`^ADy9BWFTrweL&BTDyB_caCS9PEE3T4PhhCjLbQ+W1HbO{^`!v>w^`#SH~`TSsw{!h%$+darlT zBqszjj;iO+>hS>c$$eNwT+#*nFmCo}tD}po8OqI$% zg`m@OW6Zg81Tpm=LxBi4e)+5H z_eVG*B5NgCkB}jK`{0=9Lh#CB^|3XWaC3%CrBO`@RN8sz#7MB9;k!ZOrMfD3?mw$t zBf16-2Yfa;=Es9Q^3OsqjH(6wcBR;=Z?&Lx=klpr+v~uxX27-QS1laBV_ds=kPkcZ z1MGI1^1xASy@C5~4#cYs8Yqpg0AVs7@_QE9XT@a0-0P1o=2PeE$og>2v)Lf%$iUO5 z{tPV<%*N)#uGD>VnAka^)!=GJ8L~~!&Y5tf5`{fF$6ZKebBI0p1A`2|YIK7mzpMB;<5h#{Q*PEm;|IKgYfcmOe*O{q642B8YQnHrUK40=wk#-tSIP_vseYM}66 zdo;PelJpq6woI)(?rR_4{JS`vi^g0CDox|r9l+k`2koX6t6%9?B zc!>pm>rck6sAj?IYBhe=gbLWEbbU^|Tm>k--lD&HSq1nf>s|dMSOJaiO!Y^lvtXsx zzEf*9vmpG+`z_qWayWEuM!vzQa`>#c=EAbgWpLV9aLl4hrOP~{ zJti=-YW`@9DTZ9LhABe-mh&(VEbHETh0Q3PB4E)NEZ z3QvFW=D-&6{2`5Zmu;6X25-NU^~%ZF@WSr;GvC<%*2m;lF>-(RVfjn&=NC#bT z*zuwiqOOlOtG6zN<&D?U8;43@SNNgV`w~mwsCQUM&czZ~Vc^>_$>zVd0lBU~`j6Z{ zB2R(He@gVtBl=T_JtJ}B4muuyZrbm`5_*3}SNc81zx|x^^3m|g)5HmeML6xF;ZO6! zObo~_e4X4|gbfFEf8`1l4)+tV$i^iyOQx>XPeSEV!(YO0@8iOf1Q!XnK+rn#arFl4 zl;M4ik_R31x`S=>`2@|h9|uWvyhHDQR2(XW(+yhY3k zNP9VZ!lUCXU?u2emP(bwMk-q{^JVF9za&lxgx%gD=h|Hi1-?5ykLwr1t9bXIcVC!L zwsyy(gfeP<{QVU&;aAAwRZeY8II^Ne&(yaV-gw#^zqqIr3S06{?G7j(?kA0vz-pFQ z|6&IR9``s!PWr)x^*Z0n)BO3M^>x{?k1uLzc}aDUI=w4t+xm8619`*!{ywkbuYk$pl`G)%NlnWIip8M4;j?K%YBor$P#Ohs|E&*iv!TSv z2N`JJqBz0jX%?Qn@cQEVup*r6DwBG#u@wJ4hJ6wX$84y}IQXpsr&=m7B%LZSaBJz% zoL%K8qiuJ--K}`IpFmO`4*U$fIA=u?vfht<^rq7Vx7n=m9Gw&kRWr6Nj&;bUpMx5g zS5uU{jy@;{`9`;s3eCOA+ff6=MoF(wMNxquyN6oK-pRV3>K{mWoH8k+itO8F< zXN}QkDq&wpaCnw)#c)5$KJZZQnY`dxIcVHJ*m`A7Is7(wtnuu28NjY*N&YL!XnA7) zwFk-dt)&0R{p5MUjL7#P`nrgIIbu&7$)}8`<7P;s;{y<@9rlATp5E`#kbbZ6`o;Pq zPNA9TR>WQ6t&xMBy%}4IBy!OF!}KTo37Oa+s?{}BJsGvv*x8O>9D(u=+!h>CI02Wh z-1%j>CJyZB`@x7_XOKjnNAQ*Q+n|b)CS2SEdu4>*hW>kF5#V6?N#mY8j2P?C1?^V7HzP(4_&^|?_o zc#f_K9%IFXaHWu&tePSyY)drZcol)m?dhun+=^h~ep7?f(M9m-Ormat853^(;uhNc zD289Xh6z(nm4P?c|Mb$B3Q*bW6M26twayhQN>^RxLfVD>S}4bdG{Ln&KX24Rho$DE zn1gj7l)q(jL3AA`^`smz+EfQDg-tp~R@OoedA_eXPDLl2IpAnJL-@t~3fSfKeM=SP zU#eF)RPId7hD#RdhE-JF;_rL#_h*v7ee*#>1`e4d%IUjg;kGl?g}!D*m{+OJf1y)~ z)w7qlUZCRiB+EtjZ(e7i`SG>$g>O)C$?8jw2TDqCG-uw5Y}-OyTla<8V3&?N8?o!v zgfQgg6$K?u41&;Q(oZKTr^5p>9;=+Y!!_=fgBa(r+or8-2#?Yb+{w?-y6&__p9PHFm-&5r}n#QXw_w`l?i3TnaIggE7ht%TS7=ayP*P7X!~G& z#CgUC({h+A`}WuQb7c^w3t#5GE`?Y6IMFn<^j}+=TvsRkNA4ey=SbwY6MYgyzcaC? zpX3>%>HHG(HV((p5KPAlV8I*qi-1M%|H!7_bLw1vxB2J7r7DYfu0Ws6tQky>zWJjz+%R13+oZZ75N zQ}3BM=bKI|A5wqJI##uS594Q^_1-AQ0})|~{cTkKRNtkdblu@i3nP z8Q3aS&EY`W=+%d_lzEU;dx4>!$cKfu%(ar^YvJCyRnb?(>mW08QqE~kEwqy7nJVrx z-a&&06*3EtUR%upGlf={AAKx%Z{9H5eOfUri{I<`F+3ZNT2y|vkNj_alw`$goZHGk zc5|6^fMFK8_r#~hN)%zV_o(Zgc_nyax$ldt>M}gX{`hX%oN}Di_2Jh0pi)fS8Fqf7 z5fe|I7%!mkGY8WIl(#)FPr&|FcVs^qSfbBXUHgge;z68@12<^soYA9FQ1OUgUE)>+ z{i_wnh6-}wGi%1jna6pcCTX{B`T!4Xh4YxpsC>l8{HjEm-Ce$)3pJaRD)dEaV12HR zn3`fWw8fPCOjN1@wOR*rP=9@2Yej7NqEW6C~1cH7L{Z16a_05~%wR zp9knnzxQbU&}hx#wotT7DjDV68iGGE_{UO{eQ=uOpz%9#3vi*YLjb*w;2C}1fFJFb zfgc^efEyj>LlvD512sBt45YpS1*`7tH@(1zpCuNtODt<)`Uc6no^$FzRXlH;T|ga# zrjEVQP+SL-*A=)f@vnn5OS6tWoLC3Z$L<=2EUtw|Tc*lnec?gOIXA}gvs}10wVi#` zuNsV1oVt=GR>BLW#`0;`%RtX^UbvS+G5B1ZapT)YD$YLVxxha=7verWQnJy^0ujq? zX{F0EfW<9R9NL-=g)4f#$1F$(d)Fy8R}Q7ao#)EkDxWeypXF&Fu|EeK8wW0aEk%%B z(!598hT7jRG|O_%l*5lf`=!D$RnRW}vFFKy8u021(#$-=gZ^lzHp3`BnCR_RF<)N` zF~t5AM@po=AK=5=amQ2Eeda=zy|m0kRyE8Np9t|*6fKPLsHEn@VgFJ8` z_J*c^baLBA)j@K0SdC`!K*_sJw)F-VUi!S!)7e}N5uZD|pPyvGm&dzSk7{7Tw(_(W z$rrLAUu=`hJ*xggdshMi^PU;J<|#1H#L4CR-anZ*NlBP7C8!Xsq@%mzvy1Ugp>ey{ z>{474Je2x7v;p8PA8yS^-y#fAitx*0#vg36$o(Vo+==`gqVEvVpGWMW$Bzs1 z=(-;C()A12P3LK0ES-;{$JrtqNPGbzv>yfU=<@_1`(P@B27Dv5yN{CSNDH?yUmrJ>t*C?JEQK zPw8QOdQ{#p?AJJN5fWc|C4v4}*G++1kzir(@ALYQ4>-#NsZY7>2mxLOzKG%ByZmTKlSr867LNVzgG4Cr+)77KlSrGQa@it*SBfCR9rh!Ki47ka|KdA zSNxy)xya3!t{)2-DDvQ*N=;KH8rXjrU13#-k{VGl`*lg&ZC!$jCv3xh1{dRxe7BCQ zABCt2FS;6(vT;mgVtSv>6C7D@7P+4jwBr*%;!}P3miBx|lNr}#JE07c&ffJOy@BLG z?rmct>2YH{k+2IKY>)OHkrMqQM2TQ{6mB6Y7`wXk812UK;XqFV}&`uEyrn$@L&QF@B%i zlzNyk`o^Jy=j&i?0&|7^7HYq$>pb|@nFkl|%52K=;K1eaewGpD6(I9`m+5C#G5CF! zTRwF|9@xjteja=>849m%HH_1K1Oj1;ADmE-MT^yUj1Mh-fYSTI+V}WGpjb@l`~?4K z?4b8i^!i;G*MEg-ADOY;^IJHo%RJ(@@qN+xbK0`a7Uv;JaIyWdACYiK_@Tk8->IPC zbnndKqykVZ5}a~7xCCwxdpdi4%B(k3gBm$c&cFk1R{$68c7L9eaES{VCgOcnYBeBr zTq-$Z4jZ;fEgoFTE{7QU5Mjjl4j+0GcnD}nK!ns7~ihnW1q`rVp5aQ#gHaMMxW7vj*1Lqkmo#L zGmFtzcLR=){rdBp2>UcDF7wifidp`+3~aZWeqCV4h5<<(mnI9!50nz=Y)|09IkSqr zbA?Q(T6f9(1RZF*GvF@O6jrLtOB();9Utv4SI*eb>E&*uU+ z*AY%28~hub*Eg;z2j|#D>AuT~pktqt>wsVeI0sd?-8~is0^5#SbROP=inC4Xm;MgN z!k)|1w4)O-?DJ2VcEwa|nqkqC(VB+D|Hi_p{VTQWsP;^$0EgcxXyNYNa^5QerRB~? zD$EK)0p)YIJ|rB6!sI25GsI(|f1a%F!@dl-NAyR0t8rDeC>G#c2a8c_ zh+W)hpE9EW0O?@dtJU(*3#Q{t;)C9cOF>$`@1|Nz zJ~+SEKRh)i2|B)Mh`GCZfq?V8(re_z5f6P!1B zhUHg-YD;T?)3PeK->PtIszy0HnG>bIMVAQ*WlamF&(DT0!oeH;oc~)NeG%vWxQH;Y zd-?vK4{l}R_ovbRhu$IHySVyMm;)1=j&gpLOfE*PgW)~LON&s9&PQRg&GzNv71Hp? z(nvL5g>ba3I!#JK>F z-rekFXYdfUJ&Qa}e@{heqa(MkD^lkk$**Z!j!PG)slXd|q&ZI<*;pv!S0H9t{ja@_ zT<`x#=i^51ACVWfi74pX0{W%W|ZzsgB zucOY5V5zTN(j4fQs??uuPywtvjWeIQ7K3A8Q&yR54p>b3HDy{j^*%|zYPoe?4ZG`n ze{Q`Qi`{ejHIj~`W4G=63?xYMo>2i}aS?ldpLGjzTm6{9$r#`}wKU`j+- z0p-`Tx|?VB8RX%+C)O(~WYbY>vneN9A{GU@Uk=N+�W79~}p>_nxS^)MtTp>x0&G zV1l;l+1IIRR9^p%lEW>kj#)YMh^Jy%4M%?Vp7fL9fEuZvh-qHi=-plk=fZw;E|x5Z z&0TJ?0xC?9WaM*`re#B$aB`E%mH*aTxl$q$54c%oR7iK$-jTGRd2!u81z zy#Ltf1yA$g;$i*jO*$n|M(Pu8$pzOcJ`Y6m)WOeW-pVi=j<4q<^6`8p{O)>yLcjzJ)VHGKH2oCAt#T8ATXI>N9?? z@|=R~IT)lN`Coagy<8A3dVKNccpgNDjt$qP>eHu=@sHn{#s&3scUk{V5;tnIpwm-Pq$!XI zNul$lQth%JX{pg9!Nrt+D_W@>Bt_xtsh{)ZC&+sAa$UDaCiYM2UA&l8fOSXj2l7LU z(4yc-w)_Jo9j_KLQQ~#r11o2Ew9(U zhRz;R3qHSzg8rR`YvPw?g4)niRlaM{@N)=+l|jLe6*^;YRf6HN7sB~}*ib*&(W0Zd z8gz~**k71NKkwgqe1Vul+t9djm_KRt1co9Lrmo%$29vWvW4*F`*`@#1$J(XUZ@!LU zpwy&Up1V(F;3zk z3^b|dtaGREJw+Ws>o-!kguEfoRpXIMKE_@((C~Ja`x<~L*X#_u|PuC4` zBf(c4B6us355P`>$0GF&93XhDc!J*|xDvcf@LgF1@0CRGUjo)N9*poqK)`R{1?O2N z#LqpYBc50UY`3XPcfT$JcI2A6C7Y^132OHea_+5WChKd8{Siipz%!K|#uU(1lS)i_9VY4|t76f#kKAoq067?Og zyouk*K-Njl`Ta*TFsboSjzUB}2B-C_jG9!0etVg#y@aTBLF0?4IJkSYoehft?X{1xl0IjGUA&!|9hJ{N zb2wUO>ANyGCtlE;Ut9@6*M!XOQaB8~wGPvG1Jy8B>Go4Wsy%8W+!5nc!V`k;>WS*w zccj)MI$yN2FYt>D+ZtlHqH+ZVuzK0sQyW<>Z4^q@LeEq;M;Y9%O%hhWR!XFSs}#rHomkWr zp$6;_y{l32s|ic?PUk)hZN~V5(5L$pTK=_L$@O=nzdgBsMBcg)`9z;0(Jx>~+cUze zX`uNY;6(7VmkFNMo#1PeM)0--e;Y&axXA>cOX?-4N9W0~gW!4Z5PYu%!TY`=_+M>; z2W}@i-$amZP8nDh_I2bn9K*_or3u}LIz#`($yoTEpUO_oqVst7CjOC`>Se#r8 z(v_`3s`)+<4Y^C@vfA`71 zzZG}mE0hHpST{W@%hVP|H6LF& zUTxeMkcNCbHNMX_3d0Vzz0#d}8U&;c3~1^z;Le2C(~MLzp(NN;aic&X1h~xHH#oBx zlm}+DiQF%taM4j!ahfH-BKX*jr=CkrUC09o%_8sX->5tx$%BWjU78h<@epN|(gj26 z)3A=siQ_G3*7@75ZSHgY zu`*$YRqFG9?L*{x0O>z+|A;(wB0r7jd$*F-@4Sw-XE({07tp*YsHgcK;7j-zx`dx$ zPy0O(K=>QKXdDVIBm52#IxmZ&bo~Oy({(=N5`M@X!WUr?{zwPmlMuWvW)Z%LHqHM) zG0kC~)Civt1lpWk($o^5RK8Q~O;)mF zIR_3ME=r&9vj)^|#ifmVSOdx&p%!LxHTc zSC>uGLBQm!i?4MgI3F>oTcUCblYCbgE1E{3vjn5QXE2G1TUQU-simXWVXK(HS((^G z^BHiwisK${sx3{fk2kLGO8b+9-IK=lJUvL=&s(10R&okE+*a=Uy(JRd%a`4`rkW0h zvwb%7SrvlXJ<(B76mDqSgpkIjMU|j`aiY@eBh}!uJ=K55`Wn!9wU94zy#}U4#dd|N zazOfeWM}<_Dj280TkdF2;gQjxC(eQiCWL>Ybu4=I{2TwRk9BkGN41S&;89}l!d%-0 zq4yE@HC%O_q`<`3Ln^}2xl9c4FInbrr3mkRUC?q*gW@+B&pfL7DGe_hwXOMnCK?46 zCJDrdn1Or1o|B5|u^|7XeZ}mT>7cx0#V^-g1yE4E_}8hal%M6YJI%q0nxB(Yf#n^A zZ>}7p`{Mz1ZX^uzPrz&S*?5bbcp!XH5L*+*lIx2`7U8Finc!gFW>bPUL_~i~=CCnz zzuVB6SUxttygB57 z7vT$Tr0Xi^PWXfbCxVrPZx~GYhXkLCEZVO_X`1hW+B7c;EolDW2+tO}!=<%Fr#^-N zt05&Zw`^doy%%8`N#SQjt6k@}mV@y`^+=foY-nxRocD(Er<&a*=Ip2H!g3@TK#uy=;+$-m`BC=&gN_1e~~T$37zsU6CdP*d)*sJx6%|SoH+De z$T%DXvR1Rg>Q+Ms>u8zrjX?C;VqGBnEDEKAUMyCi9C)viVg4Y7k+x#UMhg|{x`o-@ zpe3$UzDhtUnOvz{@oi({Sha{Eg!pY1rr}F zUe&e+DLl4z`#cfXB0TrZ)oO)BKAx4otGG@w9S1_Jc8?vP)`!$u7XW5 zV9}`|zX;MnxA;o5xqCj;2{V^(cvu7?N1pj|8=0`q?o0BOZPa?{b&5OBL(pI8*X$FQ z359Qm`HVn#)RaF!^Bb*GophH^$VIz8Nv);FN-^(+htHw*YV;@;xeM!R@z=n(Eu*$J z;=Q$N&+BrV@q=o@^}c)0vFz+9zv;T|7*u!p{^id0f9*DM-G%h;CHIfWiz4z3i9T7P z-;3BYGJb?-i-b>zgg1qRA411H6knX+eUacAk>IzG;8f^-6#tym9dU%eO!CM`@?~^C ziqB5sA0&PrlJgA7x+U@i1a^~kO!(7aK~y`5dJg>(EMp4kBasFFXx|+tdEiNW5mB2i9d$NkIFNQ z1*NHBJ#LYmQE+EV|e)dvB37u6MS5zQ~Qg1i;wRPyNIp(!o#h>Cas21T%5?iqxtlZ|0Qk40?oI2^CiRCpc8SbR$>VknnL%9hq>{9^~&VVryYE_`{~X8_M?2T zjDB$L8k+|@4z38Cbd9Q~e`8mLFRzBjuSD8DQ9Psg^U+VwP<*`PMFYJbDc34^#zPfEQp<(_@)lgua|+oPGWL&f0v)U2rLBkmC34 z+J0J2IgtsXf@i0S%%}1o{=Gp%R}sGZI^VmwFB57CK0S;47WZ#|=@1AG-R zF7EBQcO}EU6qypGjmp2O@xr;E4NEj?acI3c8tiJsG5*hdM?Is?k0nJ@X8J$J%~@G8 z8N1q%Z#Va5(Yto+p0Hq{{LXfCS$|V`s`qo;SSv0U$!kWnxsf}{sq;rv@%x-8>O9gq zA^t0iI-hJ9kEFLy=ap5vO~`8M{JKy0`pRUzWc0-=nFQtH&(qgm8O_Yc4%Ipcrtk+Z z1}&}3`AjsI`J^>(Z5jGdncMLamAH83Y2)=XIcTb!WX5^N#qXZi<1^0iF=@-{Mf+~n z;{FSrIq8CR6t36)2)mBTM{jo!z1h#llP%-X<17z@w*Il`$mgID-cl(KufUqCm&^Ay z7o$9}r%<>^h+~p~nb2*-}X^92TI}4?5Q6X4-~=GF9U7W85FL4ebEZsl?%7O$;-XY%Z8Ej z?7_!T*II96!*^Gmvo0@lU~rqD+kkihT&jQl`Ls_F3~X7oGx0?U41~V8_~!`=&aD~a zwa1SQ?C_}jDpFhu@ACU`Y6uUGF#m4Yd{A~fnW-jO3r~N9r>4K=!yJ1HQNuBOI3XkV z`N6DxZL&W{$YiG}P~Slq8NZX)v{IV#;_)&a zpP`(WP27Ct-n7&`$C>R+*eT%B^Ta;r8FN)AqLGtS`3? zn`E_N->COfKK8X@be&?Ju}U*8+MgsnGqnMiK6!sJg26|Q?Ut6OHEcY%bo_@6wk0S| z;*UwK6^mz6@sJfcR{{nXj9ZpkitlvaIaykk;a5fOoAS5ixcNnW#tE%TTwyBnDQ0&y z{`j+zxA7nst@>SZCeG#KFGhOj*Q8qXowNRw;paLuSGjaFEt$$kkDKtov!fo@zY=&Y zXIGB`9`0jy?ykclQ~iQo22*yi+3Xt@T#V@1e}2N5Dl{c_YcF)wSHe8&8d*@Z=+Yo?3#6rbjpvyfcM7 z&83UsV8^7Y8z)P^x#q>o-`-`QR&NrZxxWH94#B=>Zc#kBH>3A0z#0f_zv>&ai>mLh zdH4LDJs;}623)I@sD<#Eef3_7wb1Sv&%eju!`5>q%)o2ZcNtz>BjI*!yC@&0H0PD_>If&Ba3)7pzs+qP);^&@9o9?MblkT)1V$@oeaCYqKgkhtMM&CNWE~2qtWQx#3P{A-t{;$(kCN4?~L2 z9Bxg{fbi2RysM+Bd^-6}uo|3qyC4p$CT6>zdXtUE>G8wTQZEl1dp71|E?BuwgO4+- zJuY)*G$3bCsbb2}XDG9UAvWfF3sx3LPSSUKj@O>3T}v8zj@Q#o+GiL(N9i1oohhCz z`1|wN>5X+wxYRy&cbRcLZeHW1U}Vn2>u)+3Kg=qzl;p3xzqx;WPT_nwv(4$4I?=w3#B=jy(`%VyuXJX9FYKlAZK?d>1fU5lUf#3~;i zt;dCW6OM>d`Gn=i%}%TdZp4X$y7?!jG~uYlY4Y!T8*$0N6~@L*jcDoS?D?a+9?xe+ z@93LShX?oB?4FRr!(QV5rL@O9>srpjv_qj9r8AkhtmAm8b#Eq;??<7!+^Rl7#a-0* zY-E--Q{Q({?>ad{p2Fc>C;V``kgK^}m4zvS{^-6r?b!gf5UstPjh7|n$fgDdAd znpK1DM){R#(p4aF?oXSnTLoP2S21b1PsR77el(`xV(18G$G253w|i5I&NMK6wAM(oIZ2w>#_oPHCFy=532yO5)D=Qv~uCk>&SA&OKGs| zYb3i)B6+@0mhDpb-0q1 zJ1g!n30!Npqs?Bu)_TqWikvtXzh^Xw`fk6-%?gYxQ^xVbi9 zlbobl!kk)^d*+x_>cl|_QvXO=Z@1m1A`hjV%tw_~JVA@<%boYHnbGkK757M`cImt` zt;NlMKi3vsi!=U+oODyIqw*VTBi}5f_J8WPPP+k%jN1!4T^q41 zr_XTkFJ4o>nU;6C8AJZ!|K~R2rb|6vxhtNbB=rFq%hX1ksot+2kwME7p!5cMZ~D9- zn2XHA)7SV?=g1kMx`(?wOVRKt+jR9&q~%e0foaT@Rc-!ZD7~iYaOC&9Fz$PW$Ea;d zP_#dMvj3EP_-o%^zW5o&HdC{wl_0pTx9irc8VIo)eLidk#Vgw1<9zCA?QlNx2x2r({Kuazul+Xq{A@N<7$5(0;8PX!?e+XUV>cTv3LcC#)};IW<%`Er zFu-uCo_{$#a@|cf7i1O6<;3>z;O{tv-Q&^rSx|9PO|A36EG~3~@QfBw^&FvZ7ZqD-{yKIk`dK_NfOX3I)|0>hZ{b!g}ZIQGxFsxMw<1e8Q}& z-(QxLGg1Ee$EbpkTw0zNmA4jfd%aWSK8l5YR{I)!A0pG@c7Bsffaa!4Uo~%1d2;F} z{U{sWZeyqVoXY`G?};3XkC`y{*7%_XiYeeO8niz%J94<+-@YFb4wY-=MPX8bi?UUA zIA14<`8{E9pTzT(Ygf5x+csiO;NdRIm`0rbV?$mO~ z;I^0N^|(XD|MF$aEf&`YnWvye@|5aE`-j-yk(l$M_dW!B#hUs!rNCmFKLHIH2;&KV zynHd1z5q5XGCgtaSvD7Vva`M~Y2g3kcYpLh_}2)LvYnHdB)v((07Uv z{4!Vvj&j@2_%Eu1`|Rx{bxUjE@ZQWL@d^~bXI$z1Z~8n)|8}=R<H6yo|0LNO z(1Mi*w4FFG{@NS4HVqC)e6tk%M%DFs#=qCMeBnTw%IcnT&NZMnozZRlxdz@miaSSv zR6)#G)838poAiudjyi0>fu1b~+9C|8djBcTED7fdIB$P@t$bT4Tp{@(`RW8QqxIDH zzus7AZdOkK{S)H7PkDDB%ex`fV{|Cm-znWT?_A2i@%!!^@;mNXF(&lKiC;R!LV=Z@ zvE27n==N zUa26zv0v5vXd(n?GnikK!v3uf;kpn_gNSgPmhdXK!kL=q)aEKiYID4xtc_{O~i-lg*xX0^wZnhH($#*!>eD_+p zCrk5D(0r~z@*e8@i{LhMjAkI(m%f-Mx9}k-c`2;9>zxKB;X4=ms1`vQ;gkE-Ke282 zT@6VuBhIx@-~CuX_}yMhCp3LqTo0k#uJ`I=8-T$Sk~7n9fMu!<*C$Igz`7GFVTj^e zedw^6y+48aj!2Klnrm6qciOn0+K-mRlzPAuiSQ*QD!gu$ielH9DvY4##Cl?NqA- z>4K@L=DI96@HDoQol^?8?q*0HzQKe68Yc__SH~+@W~4z!#X#g2`xuZOQ@_+k`!wjv z<*N)$i$K}%QHPgqNyQS|kUqcd`TxeRzG-so6H^ve?lDfEdY_FxU&HQdb80ZTY{m|w zHZJ-veIwmHn}^e9PmT$F$VCkrFO1!iQf1k4Ra8Fz&Q}$$GW4W*WEfg_*!|V*Obj!4 z6l-b4Kz07bg@%c47~E_)H)i8~iWi(FWvvwh+s3h<{IQLPKkpK1BVyx0&u_WWf>p5~ zb#`Z&5%pb%zh}qa-+>mXQ#VQ;;{2e1mxb#SQ0jZwx|1t1aijQ~sBOy&QJW&_O)f3L zX^$5AmXE5yBzpYNA!Jo@&sPrSoRy3?GM0&Wx*!k=gUdpe?@a-z?y0VYviTrG^23Jy zfzt1jp5s_;=kSUB(<;Bb*-Mgd$;|*$3FHS@9#LaKc0^Dtb5(lTK9e3*Li-w z-%pLB%H&2oSaUq5xuyZ@Vt7G&zdyrqZ2gL_uPq$g@f6z~HL1k7t&@?^lqDu6}(o zB=w?EO6QwsTD}MBNQFBu_~Q?Q#M)Z6r^Z9Q?nbxFDVeaD`OD9Eg#Ugrj1NDVzuP8i z-kCYx6|k*l;S=u(@8tZie_H-R)Yw0ce3;jK+|+Sc2^2h06WhO&=s*TF+b?`m0IDq) zOAj2)2NeS?joA`;kmgc&V8)RgC|+LSx{TyPr9AVKEIdQ}>7=Ze_>&74$E}WaY)ytx z=I>TdZhchf_!UrDPz@ug_?(RW~se;I!AwLazWrVOv9*lc%q zF2m`E8e|*ym0}h3pWvKf8($7LDZu^IZ;9n=VsuAzlKPkLzO~cm5*@*Z-mY#diBHLZ zvj-Edq2)30kR=hP;q00t2Ucc!!^J5|(vAgwApNW1r%&&@VDsbc)FWZnx~~u2smJ?k zXx~5uhvXqkwgupU@d-os7)GLf&5e2UpU0sw8GauePQ{A%H!fQwr(>GftIFqNd00lz zAF8(~Xv>Yt#^)BRmW!KZVU3-$+`?6)&cOU33FQ@UjkDr0c-*%GiyVm$(k9;sg3qDe zy10f@7XIKjTdVy-B^lpV$^TI>NP|y5c|{4IN&XD$s|4!?nMm$#z#k8G->P+OK=X=0 z`t`*PSo!l?`b^nI95c08&U<+yPCgaQEk4qSQ8LHp>%;vT6x4X`e=X+HClG2l@ZX z+Z0RGrZ!?LTkpKjk)c_mp5cL->EWu*b@=7`y8(_5YH{QKx=91Q$a7-a>&gg+DwNru zdOc)S1wIHm*k|ROGLmN*Y+p>iN0!@+BvQ2Fc|tCVw~u+?HkF58ldF=ncBNu*$(|_B z8mb z@#;V&G)l;7tahw|+jrL8>s46=tD~kztRUBmc<_P1kdNg0&~95fqp1SMt6BGNCG(gJ zrzO!Vx0Zph?^yY&D*Xc)H$H0*a9k=`MDltXu8sD|hu6KPn&{2Rg>aH*WlwUo;-2*n zmnYZ3fLn4734_u>Dq2Ls=yM9FTMeCae0w}3xQsN&w2y*|%-Y9eP5mw@ofx-N9}DqJiX|<;jF6M?(-)- z{-&*xIv+dVxNi9UtQ7bE(f%2wRgRNa=j%NjUV(nI-mm_&s{-HDR%uI2tU%AhCP#;@ zC`Z?w2mH>F<(Z!mS5jXx4w$8DSi3YEWriQgnL+fJag+I5=DjEL|5N&>LT$oOYWb(g z{$qX7uh-16y6+F6^Q1XHxAt)cW36q`hb~=&i>Y$j^=hZOuMdm6ms`d@J%a(>zO8%x zZlYhonH_rw-D41ajpUTyxrPc)H5ygNm&2eU-NZmoHlHT*hg?#xCjY?GtLtAL*jRV{ zBl^`DUH>xKc1a$J;Qd7Q|6{;e@S4N^da>n*BU|wQz0i-okGg{O#9#pZJPP(tg!seh z`cbes>ukI)cs~#Mbbr`#d<>%gePsT9tfBsWWd41++}TRhq5gej{(Y>W{(Y>Z{{8Ot zEn?RR^Y5c8d+t(W^Ao}Tk$JoUt9RM3y09hdpNNFhJ|jI2^51ZQZiQS&0(}=+QXM2Z zQ@t?VuCog|Q~wDXQ=gX5&xm@|_sjN=lrCtF1O(H(709KzM6ihFcY$ELktv`;^B}>B z=B9#?K#!OQUuh0E2=DWzZc?jG+cw|=ol_p$jy9llfzikS-v&Gyz1#C^P6L`Ohz!~D zr2%E%=0wcv(}>MRFPF)Y>*>d?T>06h4c+5Km|Bn+kNXVgNH2=-S67eIU3pebDRsF2 z&4#1#yPo1e`J%0XEu`M{ED&y3*I@49c^+1!RXEbZHR095N|d@*In*ex91F@OPMK<5 z)~!=%itju$(WV3kHOW`-Nsj!%jG8+&Q*!Wqk@(Ni8pI#qD&3+@blA?ixcm9;YdCRP z#Q3wm!LX?Jk69&eQeY6h-+|EIZO;7Yv&Q(i%nKv+svrw36`4w+16=V_H?|U9dfm>9 z^Qwee{S48EbtDI(+U&T}-E#PGZNM~J(x;2tVemV1=D#`>dwRcyjzNO^Hx$$RIr!20 zJJ`_sJv4d>?*E|YA{Y{)A0*m(($bO{Ox=-UTKzP8P*w*Js6 zw8~28)jYilTX)R0y!x~f`-(X!4Ifa6Y`n!M>^hlE{qfkg`u6%iiwm%ARrOg{%WUj8 zGg0imb2>^Ekl@6cMC9wN?c>TLd4Y%bR%R>?MD_5%+>M7kvEATp%YCo2-M?qOJMO>c z@=k-kZ1{!m{jY;xLrct@XfKdUOHuwf+YiM3rK|U?djRXq)(ubE9|)0A8_IRdI3%B~ z-=gWMVPHwe3!;~sJKsto7L3_=Z1C>xJsJIE@boPX*cgIfsg}S0mu?1#IW9V9OX{7i z?$aM?IOTwg<=B5h)$_q7`Ayjx;uBIyzWD%MOF)s$4_Y{z7ruH_kJSpPkCo%=u|oEN z*Ve*%l} zL*F(AOFeyxR=Onv3t!ZtPEqtw)0`)0w_$Nxi(3sYT;;d&<+5u0y;x;;Q?DwlJrOi4 zFtY-SedcL;+%88&^H<3?$#bw!zm>YW^~M&`Py94QVr6Yg5e~Tiw8QgI9{w^I#&s>u z#DI3$JF1n5*i@_eLUwBa>TwDtdcX7nW%_&!2D_&3*nTUs+uyytb^R*0)ujH!{OJkT zzk6)xT>-uHmrRNv^Ng7X^lkl*Rludi2`};j$|1FV>EtiEWgxlji;aaHAC}MB>HjDG zU;p%S`n(Sp>GMCR)A;}x)A<4DSqkP0pywu-KcF9-Phca{Hzm;d1{%}(2lDBB1RL4& zHJ{E`P?OGIP>arI(1Xrzu!7EaP@m3!a2%Zvb)Ub{irKR_pD076Pf9m9XGy&$TgT`h z=L$S5>2rAD=1SDmIv4}uRT$vxKKT&wRp?bJ>3rH;g=~I?)=I;l^%tweIyRoDF0XoZ z{becEviV7Di|Cc`=6u|9@6Til>1-^U@nifbAU>Ehu1uv}B8G?`n0{(q6rO$IKGNrR z03Lg0KgxTCS!}Hbl|2TB%kEqT| zX{LE_!24R1|9l|d^=mEOSdv*#(m|esQt(QS1nIjJz22uT{S+0>W>qBge~QA-B~7SY zpb}P#Cf@_cT{5i2vj)GUO!J;#>CQv5=8k=W(MNYWl&`D7gEN<=%yFp3(iXqwN2jXr zu+u};2V^{7K6B)*hKm*GqFCWCMXrlz&4?w-9+#oTE-^jTSEbz?9=qJBX98F7aaZ`A z#{p8s*mBkI(_5W8{~288>NLULinL&{_gM9KV%)>Gia&s=4$HX_*9ym#gXnkNnlH+PH=sGc ztnmuT0j&P+IYff^s&AP#4kNj}LS3UDtrtP7pP+sOinN{t&iw@SC6vk+)SFO;)}K(F z`MdpSeF_iKdKIeD`V}6e^(;I{>suJh>NV?V{R_|2dKj|$7;4gb8S-iU40CBct^52{ zHrU)reO-#5oPPD6lv;+@^z$}#J}AfWX_HRBxLkn-v-8vCH&&wlO4SDe?Ul&td8n}9 zwvWw%Ds(J*&WX>g#9Er0gu|u7rhl+1M-y?onC)()Z^!CgYVRb~swDC;_*}tcd6_Ic zEt2D~U;-EWt}a{@`#TO*ZIcVV3A=- z*r5fnpxUUBpqG>c;`T9)*LHAW+r@N+LnMcLGRs2@&3!JrA&BJh4ocbZ$si9r*D5-h zT+Ii)=AGP{Lj~Z_=I78NQwZ$*fqTso|)%d<^ z-mshKidp|6WZ>;dm-VVK>T&5iWr=Fspg7m=Wl9xlUC=PiGOa?*<{is<<&~(gyK`~1 zZY9dre2IwnsK7SK%JnDyDaX=#fAoh;F2`|F?Q6CkDnsKVhy2dvm0|_1t#+AQioSDp zuF>`4qmWl6yXC{obsvgxo8zCs$<>9Z82eLwx?LXTHOzH6evya4hD$}XeaQ7e>+i^2 zAwF!tEGLNFWyM+H8wHZbK6)P7%>^Ci@3#6-_c*V%5Dw_yzhZlmj0e$vQKQpJL7n;6 z?+?82@Kj7Gj2Q;+$6VmUy9w6{W*#jf`N>JurX9JU*SW04@DtIQuskb0+HU}^yP*F7 zdV2-^2yCSN2{fkt3iPG@3-qM@3=E+C4U}3W=y#wc?SG&m?T6qe+8@D9v|oZRS^d9; z_ES)n_E%7f_FK?^_FwQd?Z`gPlp z`E|);Afs4%klQkUNS-%lbiQd6v@N#``1vgnTn3E%HFJDAjPo4ka-`yo}@s5wy3r&)PbT*^Xl>EL6p2ysjTEhqTL*DCd%kqJp zKTy=18l_P|{QR?g96QX4p=seG^Gnl;!De}*={VA_YS%k(N_j#Fou`n#VwBx$_if}a zabF#sYEq7Ok2%?$F)hb4E+L!WlD~ra40->4SO0H8Im#4zo*q5199M+8E}7R^hJo=g z{7+UHUG7*JCf!@@u05^{<0dH#uOvE>WhCEQX;~?b*l#y?VJYoj62Ca!GaPj9U`7dQ z2Dm)`{=69DC%t+sEnAFs*I&D>wqczLIk3*g%|-#>;j@3DsV^XdA?`FqRVfA-k!A|x^HCBB!6(d2xH zXZ}z=3i62MX&r><1#nIrG2}ax8hCKWoamUsgRbwb%g-K5 zZ~@kVB>!a0An64!OJG6Q^E}BR*Hw3Jp7fg4OEk;CH}i4r1GzGIdTREoQj*(Y`*uLw z&K5q1{d$snXdL~1yYviBA0?X3%_)NHP2S_AmJxqxYGA)UZ@b2w@z29X7J^X!UT0O` z6i#&Z?wXfv+6EF`%8r|zjf09&_`V?_;y zF?&GJ@vBesP_{+Q`=WUss;Xp-RC|+)hAWJ>yOI8&&>y1zJZF`;M-G~7yW}@jBO4P9 z*DrcY>Q{wS--C-RMvLtG8H1L$OvX(}@I?{Fjf#bzyusK=>6b})EXZm&Zg-OY*WX?2 z8XXnkS^!cGyg9l_#h^7|%rO_@Ti<^nD9e8+AHJ`s?|dxo9&y z@8`v{c{n8OL-bUI0z5+XWoTfwI3Z7*kI|#;bj}H>Xbpp8ut! z(ppr3e41B@QQciRZ@Kv zv^xB_*>70^1ju(B+DY^W125c~6k}Toja$x{&$~|gwP&8p9-&qak;(7va-kePox0AoZ|F6Dh?<(0FSvaw8(vO?2S*Vk7pmV8n z7Dfh480%%1h3!fQ(`}5i2#52Fg6fnkTqo&nBl?GjzRUMzCT0^}j45oZ+>y(W@0ZlwD&)mi3LSh680_3#Ou;wt{MX~@m5jnrsBGcZr{}Y z>F591V4i`C^JUMz(@e)7R1b=#9Bb9Nl5v=uRyjLLIv7QMEOju@S&Hkd7c1_!l&*vk|QD+aQ$=F`$(ST=p+ z&rCK=;%V4hI9dA97`|UdaazB+OlCKw0y*2M{A=nkK+-G2&2f>zO zx19FK1}mnwuDEUAyK@ZTx3tCD+7mtYMCP|>&UIUEM|@kt*IWz7=K<@JvF6#VDavb; z(Q*55$?#Rlgfk@iYPfzf`YkH=x0#iU##z~RKb47Y?ZM^B z8ocjLB04BgiDJ7)))a-Tx?82p{@SIZ;ZuAl#2X3g8^>bi2^>?5;^?RTd^?%?&>IXp{^@pGx^^4$3 z>L0P5R*)pN6BH^h>k< zq~n*CA+4EPvrwgK;qs38d8l@;aCqnJB5cz4yEDmwk15_4R9xCiv4!Rz!W#{s8oMDFRX4sZtltM|%=$CvPDMLt$o<%sZ@%EUb$8*twJFRgU@wZzr|B2P} z?fqtJaJsLLmfIe^GNut8D)nE$3>Pr-BK|7x@AE^CZOsLDQ3=a@JL1RBis8%g2v=zO zG?CTBm*9TGH{<%Sa!^(Oc=-_-H(d9t`-o(gLBwbR4NYrpm-8-|3SWqXe4 znf~N+Bz;m!ge-qeT7SlO4o1$Mx^d214tX9=Y`R)U_(fqO-#;kfp!%R4C%$KJFf{K) z!jCu(s)o0J77gX#_s_}ZEADe}&MEH53@^fyWL#oR6!^{1>6fGWu=NkoJ)z&}40~@B zQPT+}kBcaE(94_G>IXJ8S0?HDMuU3E+~e8@Q^D@+;G#%r9u)5QY8ZsEMNe~x6rq3aeByVv z(Uy7MkNDpeA5cHM&}XYh{d(w0{d?$6{e0+2{e5Ud{eGxN{eNgm{eXCk`U5eR`UOed zN#GwuP3k8^dFn63eCjtu8R|bob?Qe%XX;PHQ0iAi9`!Hc0_tb%K7Yfg-w}83_g!>& zT^PzeTz2pG!f50b6e?SOCc2BS1(tT^h>~S8^HhFjV%fLgfH4Ph@rCzz>j6oHIB(hD z9mbkuzR3EFQ_~-*NLrNP7*W&ZvmDBBp^5L^-cL*M!NmB{kKOnv7q3!pMxM7d9^O8% zz&;msjo$7|EX}}*LfLBr_a-9q$9G>J3XEU7^OTtSbK)a;duYU)UVM@-Flwn%8hK)h zT^G1{KhZ%u`kfnQNyd-P-m`CLRhB|@ZQJ{fJ!N2{B0c)*YLb7g{k!)oms0T5zHT*b zG9S{KM9-`ud|=P?nk~hfNIyI@vUvE*OlX~B@Y{M8g6B!|`PszZX%`s%xO8eX7%a8m zZY2IscK$#(zWs0i%Y33jKb+9_BlI`W;~;)pt(jt8$nF6D50Fns4O5p_c8fev3JvDqpj_N}U6`;-cGv$8cbwsa*Xj zX%1*7bj)+=&jHy%yS6VDVo%Uu{2Jx3%bn z@yuwPzy4ABGrK)?}iFW@(*&SS~t4rq4{jLY#ahZSC(ZCD>%5@M7^M!bev% zDOtIkkE#lBiiPirF~}sQ-vCl?T(L&)U0+gHkv`e0-+VHERQmercP)8Np2406gt-=3 z_vv^%ejClF>7zsQX*5|r%|(_^)4!VL(|l$5H0xMCO}`~HpT>&i)5w0L`8574pXMCP zr}@hAY5XN=K2434mLQ)-C5PtI#4MosG;6$QJ`MAq!zJVEo;vH3;U)7|zl+=ze-@Hq zoq>|Wn=#3dBtaUJUy?w5+c$`RoCGTC)8D+eO9JT|`$B6*B*6>^>$jKiUq7}@dtQ`7 z??kxNkoIW;CjlHr?2VBzOMqR2=9(`Xm;nD+Pv$9=#=}CDf#XL!h=-_j-ixDW;^8s# zmrISiE1J7A9$4N6tY>`j<1F8)g7L){FuwQ(#uv|HeDS_L_~NlW_~QTc;EPxG;EO+D zeDU`^_~L^Yf4wJP94Obmn=fueIl-tzd8j0ZM!=m!#<#aSrv>?DTUfrC zE6X>tV)EHyg$B z&DPQS75q?XFnD!?S2FRuF+zIfmB$AWe` za)`;(^+LukLQ|GM8Ay2)7(;XSQIY10<0qP< z3>7pl7R+caCZw@?PcY3ngU>$%`B5PJeoId6eb8o?1$Q>L9{A+IgA45Z{ZnXEG44<* zta$%t$*7hX&@$M(yU%=oSn|7t>kxefzh({d-#9T69Zk&d+ZQFHNp$5akERSP5sCV{{R z@-dt8Xg;PY%g3DkndW0^v3$%BmXBG;@-Ew0KBgwi$7K0Jz+2I0dX;!FY<=H3(Bf+m z6h>@KbV)9P6U={Y+Vy-`M&_$aYhR8XT?o#LgRONI6oBs!x6#}5^P%6)Cpje~_e1E% zZuOt8qSHGEw#ppkKlqjfXR@p=l>FjBSf+aa*9trctz5gdk1e^bn16hxd64FD=M1<# zMxyc2vJB9|KB?0QU!3JLfeYh{^B7;8%lP6^J^12x8DIQ8jhbJlL3wsTt{P;r3ZNVXwPk}#Z&OgM_ym2t2xyoR~`UB=PClhpO9w02Gxo05! ze!W_)md|}r1gF<{8HK+sgaXO(eY~)IVE%Jx?*JoD^$h4pPE+|EodEOKjC;E;I|!sI zew zEMNH?%U3?Z@|6Yi6vBJu^C5kpVL9mjR8X5WyBxUDWdR0~q~F5pogbJ&xnx~_>|Rga z*N-7}Gj)B|}(3?zB`9ib^CZ-^Kj0>eS7*H7J(R%hw@{QU8@?T}Tf{iCszaO1~cPryrr zXR0wI1=YnKy?eDaMQ|KldWL!Db56)g#A4h7&fP+-U~HB|pFJba;MqaeL;b!pe=V6O z7qWc*4=kVm56kBtm-&C6&mYhupMOD*eEzR2pWlt;^Ov!Fe#;*D{O&BD|2xa)uj-M{ z?@jeoaKQdiFMsoDIDRkvRWnq>9?8mMBX?2xg#{Ah4OSMlg zZZ3zD#<2E0xqd9tW0toQe(zLPU-A1ppvF8Z}S7y2`Qcwcw_ zt^Ka>z%aO1+FNashm(DyFv2hcLgyMBzn`5BDL#&i7r2q!Ddx9UWPI^=j4ytX@x@yi zU)+uH#hHH^G#Fnzn(@VBdho^jF~0b8#uqPSeDOra7jI^KapxX<@s)q!i+AQK-@dki zgE@Nh6(u<8g*g`NF${Yok5Fuur!=i_S!m1e^SIi1tZ%3P?4^1AZfItJvH zl{#}8-jJ?1pae$UbyG{DE(m6+30Sc8F2@l zFK4W~y2K3qPJCG8A>|MEOvZL9c|`-Om%$#!cTr({m%1K&m*tG_GK1;krZK+DD#mxo zXMC5hjPFv&_%6nb?_we3yXa89i*hC9yW}&zi_|X4cX`eDE)s7j--Y?lp>_G_x6UtX zLGR#apWRpsC!&}B>9xHUTxB^U)N`M}k{|PL6}_kd`Lyj*4e2jk zoL+tEadjsTm)zV^cYJgX&ZV4slw*GAP|6*~LdsXgK8zo)M)`X1iE_a}$nOrJoEP|q z@+{y3&Fu$&nokZqn!^lBXx^+aUjSUj9@4U}u7%)VT$x$hY5^p(a!L~3ySSaIz*)YBJbH)d&V0^F%j1M-4@xh)=p?t7S zjMt^k_+SY=_+a95C?716@xeM6AB^RfLfR=+O}G2?U{Ds+XXC+ou-ehuKc+YNduY^D zL%lk1mUy3?tn?JJ&v%S&k9Yz~eJ;O~`cw_unSa~h*X%+2(#k?W*leN6bs>vy}u zV_^BOu#NG>of%(TpYg@*7+-uGWrJOJbqdX)~qTCmFMfn!Ma{gf{%`1mM zn#&Br@Atb^<8?#NdZ@3-zPom6J$(8-ye4;M9k}|3x%7f(vYafccrGXmr*9>TG$&J|+4f>Wdr}kSG z3AH~8uZvVh!quwjmTR8JK%~y$ym6xvA+z3h%v>=pWS=(Qbz~3^SbqiLv?*WGit#07 z7+=zZ@g>hPzNB&wzT_n4-`T?Wl2wc^`H1l)k21ca*apg%^b_(W7gN5Z=Sa$zbZ30Y zvy3mvWqe8IKL_2-W(glRHNfDHqAvm`Ho$`4^U9Y7JcIL^Q>A0m>S2Igak{yE9jF}W zJMU(4Ep&YHOZ^d217p<8zD2C80yXC67WQdUncusPGf*lpHV4!vIv$nlAiO5n@I5Xu z6=b?%n_kC(%xnyp6BY%|^P2UZObrFk2!2N6fM8Hmm?;;sj`+lw4_n9=*JFI~iHtAa z-h(gx^)Gz!xr_%Z^q(JOeDQmXFW&qYzBq^R#hV#le8XS(;+<#KZ92b{gXa$C%$;MD zfU_#H*Pj+kN84+DcaofV2&30Co*v@<=W#S=9eyH{P;c0 z5BHIBC*dLG>p=$Ph=CX7B|!$|!oViVzkouT^A9Izp1Ckz074%otA?y?0I_z5MPq(G zgITc>#j}S$gW$?FX0<_e@WOFk@8yxTkl0su)kpmr(E6^O>UOgdmWbAF*vKW^cy|8u zP5&urPRj>L=EpR;;2f2kL3AKb|15pIfeZS#&65L+(!la~c-(2zG>}U_mSDGv3sx1w za>@>8fc@;D>#}XKV94BX9ix|obukwn*5YdpQUA(k%Z90DTOuf;?L-2suv zxiQLmmqA2!5BJT5eT1vr1{Z!UVCy4mW%=krzPK~13-;uT3;X9nzWD#Ne=dA3?8z5D z$?_3}eDQg#zpKvrz4UlRL~MyaOMVFlqt;$4?X@fcO}F;nwY8m#(^ubU97yUpg5{*L z@gHN9Fef>#o7dh53f0O)u2Ipez6(>4aqa1HWZ`DYDF()4g(hVIcM_T?Uyq)bglb^C zB*MuOaABZMO~AhZQ=0S7&R-)5H%wbT?`s3t-~N+Sey0K2J*xtvo;(A?%|!TryPhsr z_Y~f*?UnX4m;zx@uTl=`i$IImyj4>33wV>A&Bvfn_g)5zPgl-{o(J$OTh^ ziQ{a`2A3{>65*Z-j<=Nu+sxtyA1TNJ3!|gUi%;c%e2C9wBwW0VTY2`n)rD;T_;8=` z?b{gNp84bd%D1m#e0%1P|1W%dW5&1N#Q65iAOBzY_J02j-~K1f8G&_8`%{;1Y=Dl3 zEpk`o8^HVOWYf?S&%ms}P_EXl9>za;w_p0%Q;-=PB{in<3HT~6aP-?$4PWAWEtYkw z=;n*pvipGuyT26g6TNV33YlL_?>(O{_ZW-^xurci9thkYqA?36^2VY$AFMRR;Zv4<=q z+C0$Ve|X$6j^tz)NJ$BwvjB z&q2|+MRE7NdPq2&9;Ugm9z4IU_KQufgQQlY*FF!PLWXz#{YZ&gSUS_SzVJsi%z8Ob zud_8sVi{fscb>H8Uj`N?TgE)IW<>#yzIo^rn5x4d4O!aUA%O^?|k{oaJ ztJ6H+I5Y(P)~HQcZ5xI5wl6JLH^-n{VZ_v#zy8h_H~tG>+^+{;-1RSf@jX5G;=X_3 zi#PP(i;w*aU;J+U9as-<$Q~$G2ift5 z2ey^f!pnuC%nc2}JjNx~s8U2+>(r+y=!Kf|z~j*&GhJjw}}$-&QPv4e39=3_rr} zp0B3OaNTHrqzr!ZEQ2Okm4ocukyqSCSHNuPlzw)e6>vAE&)KTX3XsxCJ2&B01!zu6 z*ySTp0Rg+NnBGw-2fu|wl@mUboMMOboT-QS;5oFRaEC_`n6dn`ri?{XWzJ-Ssnr;N zcePCLCH3g9#4nfN;5P17Vlwev-_zK+I059Wmd9pIje{C~x_al*$3p)(s5lQ_J-)FP zQj*;I>fEb^x@+OL8$@d1-tMw1@TmrllqyU!Amhd2bsW9Y)s-NuA2i&)%`5HChwRAr zZDC*YK&S)LOVZ%1+LQ#5j%!C0>4bt_xgcK;oMm=j-Ka+VOQz3f_uUnak>22GeCRPc zPKhKewnWq~(LAa#i`36wtqyymn~IFPf)^QITGH!{AsqCe$}+p~Iw zBIAqiVSI5e^PfvHzBn?z_%g;9Ki`8d?!)-vA&f6R?=O7uztyiaSp8}#t6u@DUyWq- zt4LPAs%G`8k*t0t!s=JgS^Y|e`K|A=`qeX5zk1E;SH7%%g{*$1$Ld!sCjbUS%%3^= zMJ=pT_?@K|UkhXUh^aUm*8=yOjdNV!6Ihc`EB8LR1~kgY7>avWLjbE!#+|A4-ubka9e(KAyeXy7 zx4+!RXClNeeL62oXL}Jit>hJ+3&@9HRzE!(Im^a)br$HCW%6AHXTrg~LA_dL5q-PY zbG!6Hgo~PeKlar~1OAuy(?*<575dM?^VQ_CJ!aKlv(|LNr4iLI^3sk-v2#`AJetk0 zKT`=C=Ey~@l&pX$to}T5%8P$Y?vuRPxP)l0b@>qLA$_Kg7q43%8R@SS@2eLDB2F7O zm-e|0BC0<|v@X4YQp_K2JWj6P?X<_}ylk-42%lusTfatZmp|cME)%m)`^81x_Nc5& z2S`3D^J^P3zPOAu<%_p6zPMZ{<%>HD`QqxdzAeT0;;mOGU)+rG#fAQJ8O9gqGQPMi zwjEj{f~L9|M8ObKP&@i z|D$3S?SJU9{znq)e+07r$9~rTaAp0EXx9ID%=#ZJF95z7N&T$dPz{~RUX2^)RKq1* zkt+OB1rYf z`BE_JGs)}ld_LUzDlzWyP?8(0rrmmo@O7F09Ino{e8q1q1kEFv;|C8cf)6~ad+ts} zpc7&C+;Ms_qz$tSe=J`DLcO$OjJDw$4?d(=>&BcLPzuk@<0^h_FNLID=dUICmV(=z z_^2m=#2?)?-I%mafhc}wff$avDb#8+%9sUK!^I0Skgu#~_$T~dF1LISzPS5e_~P6ieDUVL@Wr`3 z_~Ot0!WYl(!53Hh3t#;I>;8Ez>z`Y){`pzfKiB+!**_n%ZM{0jpd8#d$HgcR*2vQ3l-B{>U0Qoyt=FR$@FZdmF_2b(ozAK!|%Lk($CFS;F1yKB^ z?xPdQ75Q4=_0gq2o1YWC#Sl$v=|g5kFjG!R`rJE`KS^@t-OP(Yw0hW#3?Hh?=<;#p3H+HpN|@FkL1kcl0JWy zd=BUlJi&01W7+ll>(XxoTsE9?t~nF9H&(jsHqL~Xp%=ZIK4$<=+wfjkb_Uc@{s!U5 z-EkY(Q3Q={6VC_TF9Px78GOtv1TNFR=n4W6^B`Ak?C<+(*&w%Z=#?XcZ>By*ZMXW7 zRB#@wJ@9f(Y&WmZ*?m=`W8Vv4=@2t`edjG~ZqEN_`sPR2G^$t8`vJNDppWfaRZ^OZ;FyB)95M3TW6AV&^GIrKCCR-}5xKu-m~J6_cbVulUyH|CI+W9xdvzv-4o(m9Zh(S8^ep7j>-kR}LtQi@4MGLJn+NpR#A6C(JW{^`P{TFjt9H}J5PS?%!Ej;sHcWyCb%;F-@y;B zqkEqrx~3C)IV)CVz#@Yy%IAsxkNM0&%K1g@1(z&P4V$hMyD$rc`bi6Yqg7jX5gp;G z7Vhx)bTEEkXtAg*4K6Np+aE}Bx-4HxfaP-QzAI00y8YN912d|kQud?BvAxkN zci$u25%&=uy+g6_?)e`(w#T3+(Z#HO5|5UymAdcdCu1YicgmUhHeUXkhK;{FSI%($ zJ6}BaFMM%lh5svG-12|O7oYeSzW9nBeDQgI;fw!w`dxP`=S95_=3@iX@6KiV-F{5J z8~5l%Mc$UYZhpKc)9(f_{cdC041GBVPt<$z)i>y_4>U3Tt_{=g3i;x(OuyU8^t;0E z*YDtl{RKC&2p?r)k}J_?`!RjCi2m_QOBQ62dG2PtX??Q5?85gQ>WMs1t&Z2|WyJ$K zzx9!G#CQ%J=kbXO>>EErDi`||H6zey&#@jAs6H;Z(zd6+RPzk}pGd_ODZ6U2pPsZpxF=3Ma5 z{PX-Q@n1I$T$#S`6%x+ygKftn5t;_;wR95wIOBDK=lQeRYgZ(LI@90V$7jV{6G?)L zpH{6(*qH!&avnQ1N5sLz-(M{!4~>Cl`xLXJig0kR-G2Ry6X5_EoswEnNb>5ZWpig= zJqaSabuG;YoJZ~*8{XTgcTn+_OW>3*f#_}U8G{2va?|DnVDAL`8i!DarB#-*C4#@)IO>dgPaxb`5<{2!Lg z{~`Q-gO!4osuO=tF7x;3HQz2?sFw`x{jV;Xpp*>83vTePwtpAb@RDS!QRED<`l zK3`l&!pL#huXVpjuExA-v3}pf65)E^Eqi{3B!XiD|8Z~MMDQ~`uh{II2zJ9S*l*jA z2nxk}Zx>HWgo}j0B2>NC4buDL0tV# z)$ZJQ$hEm|_T*kXD4zB#OgSA7?#zD_X>@5dpZJfM@4VYj&G&LRq7qH=lCQ43$n54>G>E z(0|TjeDP_FFTRHH#W{>GzJ~F|pE15TGQRkMzwpI(GTx~X^AFzb;UApB{DWsQvM-*L zEUaT7+kL8#4gtN~+s3E*5q`pk9on0JaUgO{effyD z9MIoB!}LNm2aHvI_q&nBfq+(2njFgkcd^OJQvx_(d+0uY?JdH)d+S)g>@o)~4s)4m zdxisremi2`ogkl^-s@||aSo(J7RyJR_7VR3pe-ruJXU3D`2IP2TF0Fq*n=C)L^ZUce98kPpTz5vE?9Z6C zdFn&Ra-ZH;NE82RV~j_pCh@N}nu$LjCrbA7Kjh#4??2Dj@!mZXSze={zR|@^V|;dG zIYY?CZ=(NFdlA^zgf90V@*V$g+)W`je)DPzi(g-Z@tnx8A(|e+xaQuezC*);(9K9J zR9ij}r+g}|-Cgkj``j9|KL3dys*F%ex_;anrH;yvtzB>(mt<@*viCpT{e5hruR)H! z4}<9PB)3-Zc~GV6124LMFs9oBAG&>TqWc5?DgJYBdjG?E`ka8;^m!Ub(0L~grS%y+ zM(da;PwV~2rF}7!rTyb>4uNIa6uU4DK0c72!tezr%0T zr;2-Nz6!ED02J;>0(yVOYeMLJ$|6i_xON5-=ivhzQ@n>`5qV1=X+%QdaR+(_bB}RRk7E?_l3(hv(F3H zCtUwWwmsqYh5IAiKjHpfW&1BY9^vr`k9Q+G{%7ob2+vO)yU)2(e>aAP-dl7345CNI zv6c@KNOGHa(Do)3!$v1$J@}Z6Q3)X*=17v3mT`C9- zu6x@#T)f?Vec001;7#9$_CwTzLNx zo@Z8n!FRMyiL9QFuW8>43u%8Chf^H|Zl!voZf;E+)vuxy)p;ZHLtqj04PiU=*I^Cy zq2f&H*G3nbYk;F@{s*$099+W2vt6`4gd=Eu2#?bG5H6+lA=IMvAzVu9Ll{r%LzqwN zL%5RGhwwA458-TDAHrj_K7^}ieF*yu5!8n;gw}^phUN?4ZdxD0;ZlP75DI^PGVHbR z{h@66Q1*G@`b^mRPq6LHVcTzJ`y<>x;r=GD{TCjOw($7a@j9{Nmt*I{mYtu)?0hkQ zH>!^?EO@D$f%l%&zie$t$IYTLXP;}QZfgQQB$;sr;m%@~qZA8&4;ot(Trk zJ>fuZ(D@>J-Yo=g-AytdF~X<&`iP^i!H&KU=5%=&Lq88}ePBn|4;FNL5J|TW3+ev+ z$N0~MU*vh5vV+yBP)N4S3v+5YOV{jX!k@xbzur{y(u9*z}w8rEGTwM&J1ybe@}z7i6Cl|GdE!{O+AoQZe=3$Z7S9 z64B<`$72M-iEU(YL#s!4o&0a*=6|n`c={UphX~$>b#!@%r=N$e?UMDuRl0r%r`v;H zbo-z}_eZc?7w5a}j>?P^&oaPz+EXL*8=3IG&3`U@o{eVDDXdH5SA(4F?DF_!kHk@cr> z9_>#HmLvUX^rQW0Or!m2d_nuum`(fBbbaLam%&~O-%n-BKV+Zp+P|*#vHmv}(SA6_ zv+c8fdH4RYema`a{(ATEXtU!J9&bmLpg&J?!Ug?$)L`dlFgst&-;IrjWU6KyOUK>6 zXD8$moyVa!!6S-pbMeFJH)B)&q@nH<^=*d=lJTTdrS*xs@#t?PGoI@njZSCgz4Gwn z;M@;lvXe%WyifLT3w;e|>HBbvE)P!h^T5^z?R5QMPqzmr==R|&-Jky$|G9A9Cw#tT zeXQ>Lu_pUoQmK9jqo~e~@OK4zRAjpCZvNju>I1^D)Gvo&)K`jXRR4wp(nFkniI?Il zn&*L|XzmT#(tIdXp*d>E@&LN)0>b%^@HtO7o(Zp8;r`8_`X-d5`X&_FFVHvrAJu(% zTut5k_K_)sj3pscLK2T5={B-KTl3(`nR9NeCeo zig$O`TF3kOeSY76-uIvH(?8GWJZGQ1*WUJ8>%Oo1y6%hX+9c*Tp(EipA>}u5d5W0d zgi?gxgo%XTgc^k3gbsw?gzkjjgmHx5gzWtbr=QvPSLpFTf6va>ot~fasqi=9SD_Of z4;vpFZ+|*|c0Wj;KfB*px}Vq5xHpdGho>lxLiOEPHKMX?xK13(?>$}_S{IA1sg7>E zq8L2-ym$H`13q@64!{Q%fpWzxDx5(n;6Fk3UZu#T-JCgZv zmwX02@;%6s;~|#(9_abNnw%e;$n`*-Tpxau@%(G_=L{cVd7uW3-&vgUg|5f!-{}?M zpCVVb=;UeRNupcq>I;%eUp))KLNklKrxQ6J<$|qs1!n?KOZUonkGu)mTUR5a1fpuLI2whz?#EEX42A#QmQzne<6R z6Vgu*_maMwlK!|1M`n1LJjDfA{=?!twx6+i%f{zP^c6_;73e_p6}XM)EASQ3SKu0= zuRuQ0SD-D?SD**cS0In*D^Ni670BK{5BiyX??#XJpucD5)28SDK(DulUf+q1hmDWw zK`?+mUv@u(=zi@<qbcrA_1)-kb4{~nPj9aIddul=pq9UZ&%Z{0&hU1IE5=g1hvl_uEPkhP3R{oa zdq?$eSWNw4b%~xDcagpW_>%PJKmqAvgS6ihW|O{XXh8Z0;y%)66UUH#v`G5_Na`z- zD2`0^c@kWJr6HE+ogJSiGQN_d>jsvO#QI(sPE5L>Tkk)9H_nA|%_Y3m;csvifpES_@g-`bz54!)*^Px4}pMvRrWli_5 z8oHnTTlD9wemkGm&0bSHp5dSj@3}$q$h9pK={Jkbq%Sk7kpAUJ`vD;B4}z@j$m%N$M`n0gKZ*-T@*mP41h>-t zjLlm%K4sD$1ieUq5Hu(KK~P{N?hk@I4{?7GFd+RwKxvz}KM2Tri~EBBd;iSoXZF2- z9?#Pzzh~#8{YNmJUT+q?zCC?jY8V<<{NB&)X;o{PxBL9ny=8`BRwC|^J65< zcYlRIzAK(YMZ82$m!AL6kU$~3c>$2{W;?~uzK!PTDLku@p*=WGQ5Z7wYoHZ zW^sx!T@TrNXHE3;D6J~)bB4O4A0YaYzM05-#@`TB9f^vW;y%n+NBWgxHtFln`QF(V zjoO`#VU(w$OY8BHx+ATxFdUiTWwR(QFoEVjEY4&58Jo9kd`;WL{iV>F^p`^3Dsg`) zP#-StF9ib9UkW@(e<>&>{iQhF5!a}8OgPBO8GL_H5eB~W`x<(@ko2bl-WqZ~+P@0r zNk1#DK0QAC_M`;#pz8=5-%c8rPNCzULGOq5&ywKgm+1X>rFGa%^!a$u=SB71Af$Ma zG{ukXDV}6Z@ueDqUqK|rpOh#brAhHAbBb57`N5}nmKMdgcogpnrubI_#ly@gK4wkv zG9`+iHBmh6-=aTfd?3cNI7;ig&9sh{LGgNqgEGA5HO(_w{7(DONY-Qa-Wd};J*tyF znJ6UvWYL`TT}A`apPc)57x(c;A@RFF+7AGiQ~nm?sjzxH?N2DFuLMyX`2fYs7%oso z^B)%Hsnh+8&097;Yto+(tx11Euzuxz;I&>bw9OUwCj<}DpAecze?nmIpNeVb+(8O) zlJ9w>pCM?|-?Q@x==u5dd8p8JLz9k&jjx!Fw~^u+CiH%S>HY1c^`vRE?rKB&HKHoz z@u<`1MfKg#MC%P_Y5l>L)+3}Zit8VsOX?rsMC%ukw4TAI^$lHG?_l%8lGa09Xnn+& z)=T)benQWO^5pzTuZMhEf9ZCD)MNfF`g42A%VYeY6_f{ow4MyKZpP{ZHz-cY@E&!V zce41M#VO`=J!bD6Pgkt32OjCO47^q1e&oPw68G&#+3m+|ZW}*FjiP!5yOa=ZZwrc~zb&x$PlJALrr&GQZ zP=9+#o%J5Q{}B2-+LoE*sC`dI9s0b)?=x4QmZkOTI$FODqxEciTHo$GuCv}9N$cOV zehzJ3lGf9uN&OsY{T%7<#d%a`J)TeN^TD)U-$v{ADzu(Y$A|yz`g6uF zWjwleqED0XB0BG*v;NEKT&zC8a7%{Ii0_xnLs`7d;tIBYO5PLhK_oMSbY~bspAv zzuoBlS5Q8ikUk#)<3|&JZkSK`P63qvWJdW==9C|$LikspLHSdbluu<&`BlD@Z`FC- z&cAfZ$FipUEMv;oQm6bab;{?mr2H;p%J+(;{4Z_F2m80^&lwM!@l_vDULE5fF`fmh zC$qYlJ;nFsDUQkT9+ua#_?^WmY&~Z0T_n-pLpt$!0Y33V0-mMVH%y`rVLbQ%%J(g# zyhg^~lA%16dRmWfqjkq-T3<=0I5NY_7%sr_A8Q)tvHgtATQ)uq;;#Wt#9sp(h`$EV ze_sfopV{}3^my7|;O{!1MfOVpy`B=yzm8Fyf{l-jmq+|Tu!Y_ayFYfn!IZxiNuP%s zeLfAh+h65&>c0tp7py5?)|m2VjVYhjp7Lw?ly9p``L|7!k84c%xhj;eOJ)Z*g(;uc zlJa|N$nmI6`M>mh$fx{Z1Iib+r~F|L$|t7dLwnbLs&AGeTA%4KU8Ea_|84qnrdMFR z?5>o5%6N8+f5dnete)&e>tF_~KEQBLir=ujwu#2?EKX^n>#;9=kAjIF9~$KRioSOQ zLrsdj4l)t4&c`C-V9?$BItiHl$FQgRjE2`@{Rae7{e~{pbEr^# zhYQtvu=x>8KBGF-hiFs1hzHe=(DNam>Pt$=^)Q&~Pb{e(<=>(|m*{$kei?UA9yjBo zGTt5I7cm|Ms~5An8N>e>&dKl|me;cQyp+Z%4s<+!7a$m%N$M`m~#!v$FW!{R))pRsw%#@DuQulK!S z5y0L*b>jC2lJDs}S7-YnjT_Zze&s=Nh-Qk{@%l{jzEhqess9+zI;j~QzdgGjdVlPG z&FTFc)90Z=pN|K9UR2)=!5hW;L8$pNxKlp}!R})HAT+HN>jxqDqgX!({5wSdXCN=u z4+8(RSU(7Y1hIY)*!-v=pE;fZ(HGL=k^Ua(`H)BTiS&BdM6QoQGM;~p{+#LGN~q4K z2h}q$9yjBw#!?<0;~&LRUInWs^Jv|S)dv_3%J3eR*RuHCiNz^&{gtNgnTLzmf0gwE zKm)2LRHwQrMXC?cqC9vH%J&^dd5ym*e`_D*sf?rb__MU`IF8m=gcL_kr+68|1=Lyo zL*qOF-Ot#(m5h)0#slpy@ONIPlKqe8OX5U!J}rtnG|=l+(Ce$y@v!kJ(ebkJ|D^ZB z?vLHC3cdff*yy`rDIyi@ntU#sB?p*Pk<;DAU7PQ{5TUH;5lC$Kz&vRmRI> z{3FJ*VE+WHZpP{Z3=oKGmJJF@x;!;$}A_3@u)6TN)Cnz;WB{wIHSxBXsKo94u!^kt&Q|4;txmoEgJ z>oO#t^LE)jK<9_85OmApX->3LYgxLj3SnG6!I% z_~Ma+uD|b3@lbH_FB;+nQ|}*Il6=A&c0X?H zc7M1p{5I1$=gakz82sjk!>}`cz95mH(-lg^74qt9q2o zr35s4d1>!dgE%x)y{@jI5RSqJdpCA(e}apGcH5;6ng;84j@b~u$Qy3fEma-Q`2<}_ zI`?F!eH{Gkl{jU={RGf3e0f_~k_bj-pP!#wl?a(F=Y3D+BP339D!b(y0WYZkIeqUS zl&sU(|A(x1?UX3ok)l+j7m|pAEtgK*+mMcvhVlGMRdcWcoNlbJ=J+kWViPB2*=Ds8vPN2guUi!DbBQ6w{=d-<(wbk432yjYcvG4FE(^*Kpc z9{b1fA8EwemXfz;-6BwXS;EbWU!S9B*6{#?=m2!4{a!5SxcU%(c3gB}c}Qt&0NSHP z7rmC}xDA|@OS~dbiSz9nErGhe%I<4G^X*_0?T`DqCZi1VA@+7OT$P^ zc-7#RlHCflQ_&L=~fU*P4Tz%@9&K!CI{cfIo!holwYF}_~rA=$g7~Z zyq8|(5D(D!#G5%b!xsibx8E8%FBtqgeIgEp!p8?&jceb9Lxsbr=*`@}>yfsv!i_H? zA@%gjOD|_~{f(&K#wXKBmGbdXu!)QpOz-#V{Ye-PIs4p~rf~hh9D5Yr-{6=6hwV?_ zlD5eJc-NSY{2a&}b$-&_r$Pw!Io5ssJQ3`v{pTgDFMwe;iw3RBD1egt3uibv7eM5U zXUkWHi=Yg*mRy~b4{;OUgby5=3+l=#3TkiD;r75lhv|KjU}thl|055g;QQy8?W-UC zogW$_%)3-|@ssGO2gR6NlYT%zneVi=s8Yz_GL`@jiqs% zo>tCi?#D3H-BLB{+p#C;JTaK(Eu#;aE}IXlYx=;}r`1!3zYYiEwNrmy+7Qq66;%Ft zWKbg5*=!6ux;F{KkDt^#`BOp!1v8By?fLlgyeVI=RD|zWRMjo% zQGg2vPF?@@stC{TJXgHbG9UB1>-3E(&BN7Uh5=hgW}~mZFnrvobhJ&VahE2}e%F;z}|;RGTiI$7n^z!+BoJi}#B_SJGb$_3x*fXgkGX>teYdku3rg zaQY>yqsbWB<~#7rk#wxutQ$V)PBzX?Y1w8qUx-~B9Q1xwapR@@sjAO~v;N#K!115P z-uAOCz>lYY^qLqa!Xww+_`BBRqd}*y`ITJUAeYfWG7)3=?{jdkGd^A zsT+s~iT^pCnrVdT7ogz0xzy}p46HdA_Os?7g3G-L-@-;Fz-Kw_e}cOtLhjU`@(q@WK;r;da6Zud z_|HVBwmO@(wJZrtsc(BiwfD=$Dd~{6f5@!c$Fd+fLo+C8cpfy}$laQ=Cm$Xx(;s3c zT>x@N<8JwH;QS(%>+=+b6~PF7hsZZeilE}W&k%2}Z@z<4p{Fc2Pd1)9?r|w2A7*-8 zef|A)E(~o}bG6@|0a-cMBabXef?-{z-!BzJfof3uXQ_a{^JD7ufQ%VZepompGIF~~ z1hV)_`1trIwiwjW&X{bul%xe*|6+pfNlCxF{Fa@#uOl0dC}Se4A3WH|W5E6aCT zGUw~j7`kmlA{0FnzlYF^tk*!}6o_4ZrClS_6?-@@_8KrhRHFO7WN~__n_mi^l6zrz zzaHaxyw1-Um{#**=yaoY!O=NXKWqOON8$l zmSja|2{GW%n8kiaaxvkI>*4-wnKY>xzK4( zelh%BrhKrWOEIjSsc@j$z7WRAwr}2hTm+tL`#n6Cm=f|bih>nq6{V@7=%i)Rx5m>7@be8FTl<2E}z4zCg!p#pS(b}h8 zMhVy!_bumfbu5-gblvVZEe!A2o|d~P`xq_1G`V-px&THk6Urmz2Y@?&&l;(zQ84a8 z>eO~ffK{VCU9&xsK~ekg@*Ja7(4_w71Ic>`AIN$Qg(R+l)!DU!^V8PShV-i zEol+n2x#Xds3Htdl&Px@%*V$aQRNYwGevE?mDSa+xmeu&WTB{AHX6M3&e7lI!1Y0V%0R{dz``N2$G*`7hoSNyV8@1}R3n1l0I^9RU^(y>*e!1iNW z5(Z96Tf+~hH!vR^9 zSEWMR&ffhewWLFGr>;Lc6AW){i+lVviyX)G_m&YkEcgQhHmkHCj{_>83Bv5qz!#_VL z8m9F#I{!WK@BElOF{l1ZH$PPDJIrz8kqB&UwEke~i`a8jZ&m9#fBVoU8LsgEY?^?( zPibKLVX z@R=Vyz&2GpzE1!1qvSn=_EPbB4TdDH0XGu=ll$fR>HqRD)VlMW^BXtTj2rg-Dr#{1 z_nM7Ccyf7CSdKzk64Vam1mxcB%?`u7+a?$PbAeWtLLUNpg z5PvDGv?(odK|ppRnC zUhn3Ya6oU_&t}I`5H@=>J-tv2KV*uwoqtsT@<+`UEXfzbEX~Z}%3nGEcIm*Uv$;Oa z*X3*u2Wv)yMWDim$9aF}hfdznu#TR7s3w^5qTzG|79Q(c)*}J2@#EPR4{lwI8JBz2 zxBM@Db>Tt9^QyP8xL{oR$vvE2x!+}zD%ExmyfDN^uTLl+s+ZZK>HKnxnXO4;XV6gpDBl_b|cC$>CH!O9O2<;RQD@!suGkFEM;q1W;2 zce^gg#vLoRe;H$zjh|HR?q3w0g*ysG^DDMyO8nh7k$H^M$$p1dNL+~dB!9v`1ZO~7 zdcFAy;}^btoPhnTcPu`;DG?3h?lcXqPQn2Tua6zMF%?UnrZ-q$%|PmJkBYj)u<`l)~3ZzHS0`DcZ%KHBecn+SSd zwePlmV+o0 zqJ~bFV7;J3SpJWEXT1c^I`3&UI)fn zWx<%sIgw+vxH#Y0WV-R8Xc!O@)o`QY@BCM7D{VQ4n-@%-Cro~WO-Ps6?c zEg1f~BLAUU2(&#}+sHRY7)t%44w3f|ZjtpGcpc)n20Tdo5A#W$47LPM0*ZselwWEp zbC<_~?@IN)YGXpdMexw7$=nlo;gzTIyw6~>@hr!#!9Hj+N$FYN8R7U|i;VA_|L%w( zy%W*es6}(hg=Bm*u4kiZ?^I4t?d3hiKNYnqM#%-AOvT>R56hIyV|-8cJCq}FA*Xv3 z=TEqg;0)+a@HlLz*YRlYQE=;F92&ejtlyLC2eQaCYh-cn1iWHr@n*Jd5>`9gN1V?| z#n9ol+gtrJaXIm;LiOX-Lu-!Y}Z_{Arng1^A0*G=0f*HGGi_n{q?7tyt3e@@QMfy2AdmZ7Z$*R zTHc$8*grq7Z+2}Ngq<-tQJ`B2N(cWeE16#e5fews6>)LfuR@a4l-`Vf3$! zH#C=pz<%m~u1nrS;7t^-*B~Hq4TO;RA8JXS3~2;Uf>?sPO8mv23pMNoKawE(yGnp^ zaXcK_8O+Pu6An!igLW>s&G|xp=>D+2&ISx(6I_G(xT5BYHRgR90wm+(>wVFJ4bj-x zSd_M;n&bD>KQa%?M%1SMi9#}u(TwbOSW4nTJVx>-G$%L%&L?;rIuP7ia{j?Keoy-8 zhhspGt$TAKUf|gQ3bTVnQ5=rk=k>S3c#O3A_BuH&5s!UKdOtZUm5c8bQ=_;zj{38s zgNcR5B@Vx$ezqvp?eti=-bJ{d_$Q*mV(0tDi;K~HOW>-$T)Z`(|Gn`25^kRC(@LJ# zy%2M|<~eNMBf@$uj|US^2=R^Uc*EY&*+~7*G2-0qHMK4wotf=bpWt9czk!ryQpGuB8MXbjHUc z%iux#E5W<56_66WK>B)qB~*qc`yBpWCGi0d>GUP*TMY{aE{*6}Q3)r$?eLW?F9(sy z@i_}dzl0f0MyrwwiXi6UCjM;B*Y4{3{T+HX+3;Z1k)){9WSH3;=;NCl4VgCwYW?&5 z@BDB)opuAaSgPS_#dK4o(z@*PXbGV zyTWBspOE;ACkzXlDkJ3bm0OBE`V=ICOUKm@o4f@OlxCiJE0XgA_<8U2h^fI~S%QtV zy}3L^!$IZarae&erh4(f!_JcNDaCvW)CvehE$X-1jm%@@k^K$@Bre3#RpR^!WeLuJ z=>(5M7lK=(CaLd8&MQ6R^MjyE-q`jh&T#82KeQk4_Q-ydXV@tFFko&%7^+l$@QQyB zgEs2UKNfyZz;2fvzRcmL;=&<2>f|2d!<+hwc)UlB9vb#}$F|L=DuX4CHtn6>3J%i?3yc+D& z=?nR@8WUT8_xM;;jnWR^`>r`!jfRKf2UK$NM7&7aR-*gdEfW_Ml;e{}&s$EqzQn$j zjs2a`uczskS~@nsvvr6aFu>zl=I5M{NGC|VW;Ay1r8>?Z4OO%Mob zAEyjUiUoa_L17ugIKQiOr~ZqI(!tPk<(`$#|N3KB?m2(RnDcoHUNUq1w#x;ubIuu~ zu;)b})3GM*&;XiupDlw&A6=p+bH2MYqKBYXhDK&JxuYo`{ zpKi&5Dp>erl~df43b0-CLslo_CCs`!T_;7Sm>hRj1o^3*33WDXmEjvpbtHpLx##X_ z@1voCTjgbWoPRt0lez5O*gVz`Wt?x!$t#V(Rp#%zX6zN90#}~cu_g({hZ!fM`XN(~ zA8tw59A{S(*(|`h@T#$RdK7kh92DNV@+movJJ^7{hamq^yk0{XiEAL3#Q)^F4nlAx zcoHlpxGT&e^$E};bv!ygA$+-i<=x`sY*;3)CuP7Eixq#ea#CQ}xTQaAhEkq+a4e{b zc;b>7Y!^n4uGoXlfXBKsX0khl=7N&ZCc!}=;}54`P;y-7!*TuJ_2<-|oE-PR=+7;w-f|sXm&MPSd;k8E{(L^oPsGRl zU;6Xh|DZo-`^S6FzAJC+tLFZFts{o4t46P)9owS%RO75k;|oJCR^fWiKX9CTC0;ml zrXzJl1saZgYT5Hx8K&2st27>2iVbhhM6Fm|BpFvE)l=_ra(uIGL<-K&Z1tJ3D;i~Y zIZPLB;`*cXTHkPd^-Yit+21E@NC@cjHY&<%#zUs%Vyz3Cli_@wwW2JhLp}9Vmx;CJ zUs@)FZ;lrOUK|s_tr7E%=zJ=KDF^ir&2lS&j?VRz^JBeG{KqIp#O)tX2hWqLuY%*( zN-dQq)j(O(+Ipi?HSna9pRlP0m@oL~*@3fvK_zVX?mgzp+A`2nT3}WeTMSPkK&~T2 z1U38g3fHXW{Oxvp8L2us1v(z>dS=}_1`f2R>0IpfcYcg|x#>;5njh+&xB0g9cLa8u z-pg-*s{r>ZJ!uMvO2TUb-D>{u6rAbyC-j$lGU{Et;aI;k5qI`*a=W@C7MIfXIE1{1 z_>K6!LoA7F;5Lc>!JFjCK=CBt6WkTfk@^JGkvbl1A@y#cJPWWiF!p$QIS;@%~Sgk4f^G~_n3x-g9hEt znjVSgG4Sd5JV;y!+9ZDh9>EzvNboq|6WkgEq`m{%qz($fgf9Rd^gIIDGin|FPr&>| z*U#NYKIZ15!;IFIu3&veYg175DX{Kh_h!8e*I!L|{i(*?r&uc3Z(n;Y0_!)2&L8$i zfbw(ZuPs}VD(TZe{n>Hms^T}d>he*3VBoaIJq0**+KrHHdkS%Kyl2y#%0e8s((S#l zxDdmdQdcQ&FT|k>%1q7A7vQ9Xr^+(=A{*qEI;k#dW-Iw437`3jqVZWeasOa3L zp1lOE6(b&Ho~i(q;~OoKva2|}%KcjH>>3c2^w~1-ehqxNFyc>l8*Y9aNLQSjUk#&a zeiAdGXt8o&8H9xAzaQ791l9yNEMNGC^Wz>?8CHHf2inITOEA_>fwd{AX0795Ak3qE zx!!=k^W&c>bA%al{jhPviAq_ONW5pdWJ-?`0h+$~AYVQ+8SilYI5)pfL8C{)*rMzd zbl>GYB)TXG^=bTWP2NNFC+jsz)r;dA5Kw*gMUp2&1;LZxJi%Sz1*uPf5~<_C7gFyA zZ^8=!%0Gg}dk=0`9ua~~i2bbi@LV|7_`<@K^Cwr|8I?Tb6<1GMk@9E6hBVkX^NfFc zGKWK~_c*z5OCp%k{D)8GF&LBm4oXQ}2sLzk4FqQZOM=IN{3|hT4YHBq`VPpR64yaN zdYhOp0DQu$k^G*YzQQ!MDGUs5_b9B74&~}#k^MeYb3FaV4LWb8`GSG_r} zh3>m$g-h@b%(1yRx1ZaD3U%g6LlbsFi+v?9HMAG%`KC{?Doc94V!d6eJn$rneo@newLxjp-B7qKIK@tG|wxmUpclp^y%r(;asCIN{pp>#mR<{@Fj^LwD#tgn6Z6pE>?ITZ{9Eau zB4?pHPc9esx7kGZ87YFMb+VcFjfx;;WLcVjI2UIE9IqXz(YZO>6iLSi4XkKzRnY`X_mm?Y!{mmDx9w` zzi6FNMh=8{O&ER8ECn7eE$}p+69awIKdafu{hc4lkpW9jtnkC^vQ%Riok%?QJs{La zJ^{UNcDYy=n~dgDZoi7;@E6PM4-JDyq~UOi_k@u55YLkJ8gocogZ!D|_#Z7vo-EO6 zCllNi)Jc5;zLGi~G?02X%p$xH*hlzN5`S?8O^rpiJB8r*QQ>$l=c`;dBtIUu<-p#? zrK*oOKUtYQYJLjmxwz}#?9sh!)1V*4%WjZ)3_4`LgGLe;f(OZ;fJbl!$@t6(ZVlR` zz5_L+4hk-WF94Q=R|5uwe1WRO zHOzPY*U5n(Fzk2r*=)|oui*4pogh1GYV7LZxZWQlCkb~i{1Aamzu&Z>#MZ$e6=R#F zt_L+`B2c`mDl~Kc)DJ=o3t4L9*_Mxfy$*KEcM##2VO^I!3KZe=EvWmCsR);DbaU@o zo{!WY8y`IDa{k-?5_HyFmA-?+aXpGh3KdgIP%2#2(r9og&aj;{^UU^AtTzb?F+9ud z7mY*g-s+a3`a`SFdc`HE^|07Cp{7{UU#LfyDE^#N1=!8~l!t%6eDr8vWfyTi8|7*L zRN<&S7cW^MrjL}@da4MoVQ#JakugH>e^5oNm0A-oc9&)B=AA-ftT zT?i56a`9DwMv|G1Cj$>>k;B)Cg(SBYMG3aR6P zOs=@z4eo>&0zV0V3epJAP~tCc*1h9FpTT+1-ag9p&==01_-b9xQ=!?gU%$jHZhIE= zUQqaA_ty+~LF+5lWFCVV+3#Q}i3>rKa0bvKcpT^w+*&feLQ)3>U&0rFeS}v7 zzJz}UUWBJ9IX-sGnc$8i381eNy5FTq0Bu!OdlOC|*q2zfPumd#HKk|1EV~>InqymU zC#wd4W|_`N?}c~3dX3*9jJ%087ql(}7zZNL?>Apm>GsGv0WBYPOwu2iij&^Us1^6g zMCxyk^HYx06b0nrbH(FUl`cXo?ZsQ+^ht=pBlDU^)Cti`C1OOlBhAmb`uWF;s(o5? z3-G*#mi^aj9F81ZJ*c3i0OQQ&9lXvb^~M^_B21i?<=*XW5q=+(duTt0E9VZ}*G_kee}$+0HKpHX2cxFd?jPR*uOm-o|HiJa9FEfY$JKv*)rO@RyC4{h`?!46>{L+SKJKIU z#vGWMJod|1E+6SUOhc;}&f1x0hg7|UmZ>ZEPx?{;2d#|GG%u_Mr75}cuB&K%ZdwBy z7YF+v`ce&r?QN^C{j3D#ocbf`1IwYYvwnE6gj|=a$5#t@iI=5vL8V7@%N+;K|8>8d zOwfiHC|Y!H`5?~6uk*T{e>e621ohwThvwz#y=|Ux{tu_#oSx^AfFm!B3q4(uf}O{7 z)|0!D_Yen>^&0PyxCY0O_+NY+FE|$4e~9rUOn)cFU9qzs!__AslhpCZb-8+aE#ZZL zIpI%10O1)zAHv5a$8r3F8wZD(dw0!-rNs)XKCaAy?Cr-#^gfpfJ43!0TzilqKCZJK zpG)R3(EScN`*v}0AvloyiJZ?k8Nvu22gU@q23p^NSW*X-jIW*WYJl?3z>DxS!G-Xd zCC6#6wwOF4C=t4?AHDVcf<$QUeJEt7jNrBPV%Ps{k>!0qFgvL-1GPC>=`1-Ua; zaJUio*IAFZ{rIU%^xym@RB9tq)tj@tY;(w03i*UkwY%V5` zcU-plX)a1DZIZXk&qJ+!<^vz43Ngn*^eeR_AN$Fy^B(&{gkE-0u~(%E(P-y{8H2J5 z(QlUffFslY_M>a*Gv?3Mqa1E_Uv@+7i~>~pZ8z|l5f}gID~~-rD;Lc|3QuExIvykb z)0m#oG;9>dcT|sBcU`Y@9riN&C$IYkWUgT8*VEUoM8-{ZJ z$VF9iCXd@116M49rq(O{oge)@ZI1Oj=!bh&`^!3(arwu`=wA~#ek}7p-%Z{_+(*`H zG$(Nls*(5~Z<9P3ClEXd!wBw*?W8_|9%14-9w?A{H)s-G2y~(S1GW>MA#5akT(Ba% zb)fnNaKvbP%eXWMtXo#_Qit;=A6l`=swS1|8@|u{ltC)|wiVA~a3lL2*pj#qv`GF0 zO9{>ZZA-*>95g%;N{XY>Y$KK_ySP3Qp~G?ni=6{fm0J9jPNu;gYcQ5(MQbt zmwaE*<@g|((TOmFzwxSGasrgDdcDA*P5}1ZRjs~reH!?yvO{m3hz9HH{2I}#FyQyd ze)`)q01R|aS*%au;;rs8N6-Io92wqKqp`KSL-KQUix&(oGK)eRUfPMp69hQ&-km## z1}EW!-HFCq3{&tn@n^?z_wT>^I4~WD);Nzkb1of6J#n^JY|iyDq5R!(<2TtJeV&3I zL))64Z%)N=6Ph3C2-48aW#8AEO&O@1TWvc%DjU7Eu9(IO^YHzPq`|R=^D))o<655- z5xSkt&PM^qkEA%+p|?FoeB|o=+OTyG94$co-o<>Q;T%soKyicv$J00V`kC9GpM}z~ zAEZzHNx>T8KaJX5$3`CA9Evs4(u2O2xN`r!krg@78I&T*(j#p`pmA;?xXlzmySlJG z?JZX?c|4|R(f}bS{FXUv$9*T?TzTJDr4)8=ZnElqvm6Q@Cg_Dvt%99}{j@4`t05^i z?Vhr74Ghj&r{Lwr?H@xQ-`ktb#bL{nvhseH!C2>?M?6eR!2RrLvy0sP7;1FqPffpE ziBEitzU6B#&L_FLGHS1I^xye$Db{3xl%*e99Qplcf>addP`>IL@*ZL?S+7xs#5MSp z#Q#Y1WNapQ67D6qE4q{V1e%jN9tMzlH)<^s^Fko~u9!asmkG}h7STQeBEnmTQAFPX zRQCgoK`DJZE+)cPlbmXoO`Kmi<@;VF^O)l;i}yQFA#ow>Bl#1YB{%~#OcUdAph|FS z&?WU9uqSm;@Fjc!a3Q=Jh$Q?o@E|-*(4^x_j~4U(!J6n}B>%tkZj0+5D+J@A<{8;v z;vqdxCUVd{t{+yA&YafeG0?{ElWXuH0@R0f+#SFV1wlh#!6P4TeoP)a&@#&dtS8QE z-P_$7SRYfRlZT5Emw97=w8gXwy3es`WA@UpUoUWe_{zbjOJeZ6e^|75ve~mHa&ICSGyO3{6QYWLi)v_YD$E=s)_&`Ha{Az z6rPBKD;|WGOL4q{vE90dq-5Zo;}_OjdE{WfqkFGB<>I$PJ&vjH`iYSBFEz~nX#CTR z(+ez$*n3W;0Dr!(n{P2≫vk$t`^(#0J$x%Qb&wqi(l_d81R)(EgS~Z0)B6^riZ3 z^C4mKcTYWGeym`w;H!{T76@$?7i+i3#X;kmhVy&&q=L2Vo5)Qmxv;czoAKoGkbC@<`$`~i?cOs}HWt8*8P|+|_RR%_QOmZs-$?-#sy``S+cJ5=n7{Mm z#cBWis0)5Lh3Xr+n~L8<%pvPFE+=sf{wDE1-Y0o7ULbf9{vx<5^$W~Eb5h5{L{jfY zSHcTHmG@%)6lxlYd4`})__$DXSIk?N=(z*h#5yR6zxY?jA1msYMoIJ`+GHNXVzS@C z8WI;mg`PNn0$+kNKsv$WK!f1c&~R5=-vMJ%2L*Y;7l3ras{v`kKZ7EKdI+d5 zY@55&ChF)*!s5Z3N&~TBzRI@O=R?p}X?o4pagk^#^JeLpD~NadUKb)dgLG_(dx z*K>`5;EpjpCxxbfSzmtYqzN2fE3|QC!}bCwoc)iT7PtO4w2mIo^&3~uxO-3OU2r7? z%=g(SbFdmTvW1poINWNBqwEG7(`pVU-nHb-(@GdObGxtChcf7@U^zI}pah!8qd zP|O#A;0!UZ2DAzP41#mTJWb%Mi22MQY!UPR!P-Hrj{&}-SVtxq9~-BT_}z1Om_lIo zO<(Y^n0hnki4Q1!d7ND+;|=_Nt~(;4xVngzU*eZwE-yX4K6OhUSI}&EH|12|O|XuW zpIxbT0hoSYu;yL!qkFdKQDr{dRqr+iI~BPedU_v?Cv5oPZ0U*W&HsFlKIw(+NqLt8 b1m381$139fdSBf0dVkNk)4BMb>KXqJo*MsC From 028fc8619ae38c74a3d3ac759bd6583ea4100aad Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 15 Nov 2021 15:30:57 +0200 Subject: [PATCH 078/161] Remove PKG-INFO, .appveyor.yml & tox.ini, update MANIFEST.in --- .appveyor.yml | 33 -------- MANIFEST.in | 9 +-- PKG-INFO | 215 -------------------------------------------------- tox.ini | 3 - 4 files changed, 3 insertions(+), 257 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 PKG-INFO delete mode 100644 tox.ini diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 0effb3f8..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: 1.0.{build} - -os: Visual Studio 2015 - -platform: x64 - - -matrix: - fast_finish: true - -# Should speed up repository cloning -# -shallow_clone: true -clone_depth: 5 - -#init: -# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -#on_finish: -# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -cache: - C:\Miniconda3-x64\pkgs -> appveyor.yml - -install: - - call scripts\\appveyor\\config.cmd - -build_script: - - call scripts\\appveyor\\build.cmd - -test_script: - - call scripts\\appveyor\\test.cmd - - diff --git a/MANIFEST.in b/MANIFEST.in index 7233b061..b7a2acb1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,8 @@ exclude MANIFEST.in -include CHANGES.txt README.rst pyproject.toml tox.ini CMakeLists.txt -include pdal/*.txt -include pdal/*.cmake -include test/* +include LICENSE.txt CHANGES.txt README.rst pyproject.toml CMakeLists.txt pdal/CMakeLists.txt +include test/*py +include test/data/* recursive-include pdal/test * -recursive-include test * -recursive-include pdal *.pyx recursive-include pdal *.hpp recursive-include pdal *.cpp diff --git a/PKG-INFO b/PKG-INFO deleted file mode 100644 index 43700f58..00000000 --- a/PKG-INFO +++ /dev/null @@ -1,215 +0,0 @@ -Metadata-Version: 1.1 -Name: PDAL -Version: 2.3.0 -Summary: Point cloud data processing -Home-page: https://pdal.io -Author: Howard Butler -Author-email: howard@hobu.co -License: BSD -Description: ================================================================================ - PDAL - ================================================================================ - - PDAL Python support allows you to process data with PDAL into `Numpy`_ - arrays. It supports embedding Python in PDAL pipelines with the `readers.numpy `__ - and `filters.python `__ stages, and it provides a PDAL extension module to control - Python interaction with PDAL. - - Additionally, you can use it to fetch `schema`_ and `metadata`_ from - PDAL operations. - - Installation - -------------------------------------------------------------------------------- - - PyPI - ................................................................................ - - PDAL Python support is installable via PyPI: - - .. code-block:: - - pip install PDAL - - GitHub - ................................................................................ - - The repository for PDAL's Python extension is available at https://github.com/PDAL/python - - Python support released independently from PDAL itself as of PDAL 1.7. - - Usage - -------------------------------------------------------------------------------- - - Simple - ................................................................................ - - Given the following pipeline, which simply reads an `ASPRS LAS`_ file and - sorts it by the ``X`` dimension: - - .. _`ASPRS LAS`: https://www.asprs.org/committee-general/laser-las-file-format-exchange-activities.html - - .. code-block:: python - - - json = """ - { - "pipeline": [ - "1.2-with-color.las", - { - "type": "filters.sort", - "dimension": "X" - } - ] - }""" - - import pdal - pipeline = pdal.Pipeline(json) - count = pipeline.execute() - arrays = pipeline.arrays - metadata = pipeline.metadata - log = pipeline.log - - Reading using Numpy Arrays - ................................................................................ - - The following more complex scenario demonstrates the full cycling between - PDAL and Python: - - * Read a small testfile from GitHub into a Numpy array - * Filters those arrays with Numpy for Intensity - * Pass the filtered array to PDAL to be filtered again - * Write the filtered array to an LAS file. - - .. code-block:: python - - data = "https://github.com/PDAL/PDAL/blob/master/test/data/las/1.2-with-color.las?raw=true" - - - json = """ - { - "pipeline": [ - { - "type": "readers.las", - "filename": "%s" - } - ] - }""" - - import pdal - import numpy as np - pipeline = pdal.Pipeline(json % data) - count = pipeline.execute() - - # get the data from the first array - # [array([(637012.24, 849028.31, 431.66, 143, 1, 1, 1, 0, 1, -9., 132, 7326, 245380.78254963, 68, 77, 88), - # dtype=[('X', ' 30] - print (len(intensity)) # 704 points - - - # Now use pdal to clamp points that have intensity - # 100 <= v < 300, and there are 387 - clamp =u"""{ - "pipeline":[ - { - "type":"filters.range", - "limits":"Intensity[100:300)" - } - ] - }""" - - p = pdal.Pipeline(clamp, [intensity]) - count = p.execute() - clamped = p.arrays[0] - print (count) - - # Write our intensity data to an LAS file - output =u"""{ - "pipeline":[ - { - "type":"writers.las", - "filename":"clamped.las", - "offset_x":"auto", - "offset_y":"auto", - "offset_z":"auto", - "scale_x":0.01, - "scale_y":0.01, - "scale_z":0.01 - } - ] - }""" - - p = pdal.Pipeline(output, [clamped]) - count = p.execute() - print (count) - - Accessing Mesh Data - ................................................................................ - - Some PDAL stages (for instance ``filters.delaunay``) create TIN type mesh data. - - This data can be accessed in Python using the ``Pipeline.meshes`` property, which returns ``numpy.ndarray`` - of shape (1,n) where n is the number of Triangles in the mesh. If the PointView contains no mesh data, then n = 0. - - Each Triangle is a tuple ``(A,B,C)`` where A, B and C are indicees into the PointView for the point that is the vertex for the Triangle. - - Meshio Integration - ................................................................................ - - The meshes property provides the face data but is not easy to use as a mesh. Therefore, we have provided optional Integration - into the `Meshio `__ library. - - The ``pdal.Pipeline`` class provides the ``get_meshio(idx: int) -> meshio.Mesh`` method. This - method creates a `Mesh` object from the `PointView` array and mesh properties. - - .. note:: The meshio integration requires that meshio is installed (e.g. ``pip install meshio``). If it - is not, then the method fails with an informative RuntimeError. - - Simple use of the functionality could be as follows: - - -- code-block:: python - import pdal - - ... - pl = pdal.Pipeline(pipeline) - pl.execute() - - mesh = pl.get_meshio(0) - mesh.write('test.obj') - - - .. _`Numpy`: http://www.numpy.org/ - .. _`schema`: http://www.pdal.io/dimensions.html - .. _`metadata`: http://www.pdal.io/development/metadata.html - - - .. image:: https://ci.appveyor.com/api/projects/status/of4kecyahpo8892d - :target: https://ci.appveyor.com/project/hobu/python/ - - Requirements - ================================================================================ - - * PDAL 2.1+ - * Python >=3.6 - * Cython (eg :code:`pip install cython`) - * Packaging (eg :code:`pip install packaging`) - - - - -Keywords: point cloud spatial -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Science/Research -Classifier: License :: OSI Approved :: BSD License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Topic :: Scientific/Engineering :: GIS -Requires: Python (>=3.6) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 1a621811..00000000 --- a/tox.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = -p no:warnings -norecursedirs = _skbuild/* gtest/* From 5dff76ebd484859f28f255537c116b4d5b8ed894 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 16 Nov 2021 18:36:27 +0200 Subject: [PATCH 079/161] Remove SKPATH variable from GH actions --- .github/workflows/build.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8aed3bf4..f9ac47fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,10 +49,9 @@ jobs: - name: Test shell: bash -l {0} run: | - SKPATH=$(python -c "import skbuild; print (skbuild.constants.SKBUILD_DIR())")/cmake-build - export PDAL_DRIVER_PATH=$SKPATH - $SKPATH/pdal_filters_python_test - $SKPATH/pdal_io_numpy_test + export PDAL_DRIVER_PATH=$(python -c "import skbuild; print(skbuild.constants.SKBUILD_DIR())")/cmake-build + $PDAL_DRIVER_PATH/pdal_filters_python_test + $PDAL_DRIVER_PATH/pdal_io_numpy_test py.test -v test/ windows: @@ -90,10 +89,9 @@ jobs: run: | call conda activate test set PYTHONHOME=%CONDA_PREFIX% - for /f %%i in ('python -c "import skbuild; print (skbuild.constants.SKBUILD_DIR())"') do set SKPATH=%%i - set PDAL_DRIVER_PATH=%SKPATH%\cmake-build - %SKPATH%\cmake-build\pdal_filters_python_test.exe - %SKPATH%\cmake-build\pdal_io_numpy_test.exe + for /f %%i in ('python -c "import skbuild; print(skbuild.constants.SKBUILD_DIR())"') do set PDAL_DRIVER_PATH=%%i\cmake-build + %PDAL_DRIVER_PATH%\pdal_filters_python_test.exe + %PDAL_DRIVER_PATH%\pdal_io_numpy_test.exe py.test -v test/ dist: From a63fdce01dbf5868b91e3d7350c496de23fb903f Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 15 Nov 2021 18:48:08 +0200 Subject: [PATCH 080/161] Fix deprecation warnings --- pdal/plang/Invocation.cpp | 2 +- test/test_pipeline.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pdal/plang/Invocation.cpp b/pdal/plang/Invocation.cpp index 380ac132..b92b087e 100644 --- a/pdal/plang/Invocation.cpp +++ b/pdal/plang/Invocation.cpp @@ -53,7 +53,7 @@ int argCount(PyObject *function) // Owned by module. PyObject *dictionary = PyModule_GetDict(module); // Owned by dictionary. - PyObject *getargFunc = PyDict_GetItemString(dictionary, "getargspec"); + PyObject *getargFunc = PyDict_GetItemString(dictionary, "getfullargspec"); // New object. PyObject *inArgs = PyTuple_New(1); diff --git a/test/test_pipeline.py b/test/test_pipeline.py index dbac18e6..8da71aa7 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -352,7 +352,7 @@ def test_read_arrays(self): z_vals = [1.5, 3.5, 5.5, 7.5, 9.5] test_data = np.array( [(x, y, z) for x, y, z in zip(x_vals, y_vals, z_vals)], - dtype=[("X", np.float), ("Y", np.float), ("Z", np.float)], + dtype=[("X", float), ("Y", float), ("Z", float)], ) pipeline = """ @@ -379,7 +379,7 @@ def test_reference_counting(self): z_vals = [1.5, 3.5, 5.5, 7.5, 9.5] test_data = np.array( [(x, y, z) for x, y, z in zip(x_vals, y_vals, z_vals)], - dtype=[("X", np.float), ("Y", np.float), ("Z", np.float)], + dtype=[("X", float), ("Y", float), ("Z", float)], ) pipeline = """ From b5990a9ec0494f52cd9f0a1b5fdd5f5b030143f7 Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 16 Nov 2021 18:09:59 +0200 Subject: [PATCH 081/161] Fix setup.py: requires->install_requires --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3aba0fd..bac5ce86 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ setup( name="PDAL", version=version, - requires=["Python (>=3.0)", "Numpy"], description="Point cloud data processing", license="BSD", keywords="point cloud spatial", @@ -31,6 +30,7 @@ long_description_content_type="text/x-rst", test_suite="test", packages=["pdal"], + install_requires=["numpy"], include_package_data=False, classifiers=[ "Development Status :: 5 - Production/Stable", From e7c5b063dd2809d550aa3ff050e381555cdacb8f Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Tue, 16 Nov 2021 18:10:27 +0200 Subject: [PATCH 082/161] Minify build-system requirements --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 82955c60..972b601a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = [ "wheel", "scikit-build", "cmake", "ninja", "pytest", "pybind11[global]", "packaging", "numpy"] +requires = ["scikit-build", "cmake", "ninja", "numpy", "pybind11[global]"] From afd56ead91932fe284386152de41b63df12568bc Mon Sep 17 00:00:00 2001 From: George Sakkis Date: Mon, 22 Nov 2021 21:11:57 +0200 Subject: [PATCH 083/161] Remove plugins (#104) * Remove PDAL plugins * Skip tests that depend on optional plugins if not available * CI revamp - Merge jobs for Unix & Windows - Move conda dependencies to environment.yml - Replace miniconda with micromamba - Install python pdal plugins from https://github.com/PDAL/python-plugins and test against it - Build and upload wheel distributions for every OS & Python version combination --- .github/environment.yml | 10 + .github/workflows/build.yml | 144 +- CMakeLists.txt | 260 +- MANIFEST.in | 8 - README.rst | 9 +- pdal/CMakeLists.txt | 8 - pdal/filters/PythonFilter.cpp | 159 - pdal/filters/PythonFilter.hpp | 72 - pdal/io/NumpyReader.cpp | 684 - pdal/io/NumpyReader.hpp | 134 - pdal/nlohmann/json.hpp | 20894 ---------------- pdal/plang/Environment.cpp | 385 - pdal/plang/Environment.hpp | 85 - pdal/plang/Invocation.cpp | 484 - pdal/plang/Invocation.hpp | 89 - pdal/plang/Redirector.cpp | 208 - pdal/plang/Redirector.hpp | 54 - pdal/plang/Script.cpp | 66 - pdal/plang/Script.hpp | 78 - pdal/test/NumpyReaderTest.cpp | 335 - pdal/test/PythonFilterTest.cpp | 1167 - pdal/test/Support.cpp | 415 - pdal/test/Support.hpp | 146 - pdal/test/TestConfig.hpp | 62 - pdal/test/data/1.2-with-color.las | Bin 36439 -> 0 bytes pdal/test/data/1.2-with-color.npy | Bin 36546 -> 0 bytes pdal/test/data/autzen-utm.las | Bin 37417 -> 0 bytes pdal/test/data/autzen.las | Bin 4962 -> 0 bytes pdal/test/data/four-floats-be.bin | Bin 160 -> 0 bytes pdal/test/data/four-floats.py | 10 - pdal/test/data/logs/log_py.txt | 1 - pdal/test/data/perlin.npy | Bin 80080 -> 0 bytes pdal/test/data/pipeline/crop_wkt_2d.json | 14 - .../pipeline/crop_wkt_2d_classification.json | 20 - pdal/test/data/pipeline/from-module.json | 11 - pdal/test/data/pipeline/predicate-embed.json | 11 - .../predicate-keep-ground-and-unclass.json | 12 - .../pipeline/predicate-keep-last-return.json | 15 - .../predicate-keep-specified-returns.json | 12 - .../pipeline/programmable-update-y-dims.json | 11 - pdal/test/data/pipeline/reproject.json | 15 - pdal/test/data/test1.py | 9 - pdal/test/data/threedim.npy | Bin 416 -> 0 bytes pdal/test/data/threedim.py | 5 - pdal/test/data/twodim.npy | Bin 200 -> 0 bytes pdal/test/gtest/CMakeLists.txt | 329 - pdal/test/gtest/CONTRIBUTORS | 37 - pdal/test/gtest/LICENSE | 28 - pdal/test/gtest/README.md | 244 - pdal/test/gtest/cmake/Config.cmake.in | 9 - pdal/test/gtest/cmake/gtest.pc.in | 10 - pdal/test/gtest/cmake/gtest_main.pc.in | 11 - pdal/test/gtest/cmake/internal_utils.cmake | 358 - pdal/test/gtest/cmake/libgtest.la.in | 21 - pdal/test/gtest/docs/advanced.md | 2567 -- pdal/test/gtest/docs/faq.md | 753 - pdal/test/gtest/docs/pkgconfig.md | 141 - pdal/test/gtest/docs/primer.md | 567 - pdal/test/gtest/docs/pump_manual.md | 190 - pdal/test/gtest/docs/samples.md | 22 - .../gtest/include/gtest/gtest-death-test.h | 343 - .../test/gtest/include/gtest/gtest-matchers.h | 750 - pdal/test/gtest/include/gtest/gtest-message.h | 218 - .../gtest/include/gtest/gtest-param-test.h | 503 - .../test/gtest/include/gtest/gtest-printers.h | 928 - pdal/test/gtest/include/gtest/gtest-spi.h | 238 - .../gtest/include/gtest/gtest-test-part.h | 184 - .../gtest/include/gtest/gtest-typed-test.h | 330 - pdal/test/gtest/include/gtest/gtest.h | 2478 -- .../gtest/include/gtest/gtest_pred_impl.h | 359 - pdal/test/gtest/include/gtest/gtest_prod.h | 61 - .../include/gtest/internal/custom/README.md | 56 - .../gtest/internal/custom/gtest-port.h | 37 - .../gtest/internal/custom/gtest-printers.h | 42 - .../include/gtest/internal/custom/gtest.h | 37 - .../internal/gtest-death-test-internal.h | 304 - .../include/gtest/internal/gtest-filepath.h | 211 - .../include/gtest/internal/gtest-internal.h | 1380 - .../include/gtest/internal/gtest-param-util.h | 883 - .../include/gtest/internal/gtest-port-arch.h | 107 - .../gtest/include/gtest/internal/gtest-port.h | 2231 -- .../include/gtest/internal/gtest-string.h | 171 - .../include/gtest/internal/gtest-type-util.h | 3335 --- .../gtest/internal/gtest-type-util.h.pump | 302 - pdal/test/gtest/samples/prime_tables.h | 126 - pdal/test/gtest/samples/sample1.cc | 66 - pdal/test/gtest/samples/sample1.h | 41 - pdal/test/gtest/samples/sample10_unittest.cc | 139 - pdal/test/gtest/samples/sample1_unittest.cc | 151 - pdal/test/gtest/samples/sample2.cc | 54 - pdal/test/gtest/samples/sample2.h | 81 - pdal/test/gtest/samples/sample2_unittest.cc | 107 - pdal/test/gtest/samples/sample3-inl.h | 172 - pdal/test/gtest/samples/sample3_unittest.cc | 149 - pdal/test/gtest/samples/sample4.cc | 54 - pdal/test/gtest/samples/sample4.h | 53 - pdal/test/gtest/samples/sample4_unittest.cc | 53 - pdal/test/gtest/samples/sample5_unittest.cc | 196 - pdal/test/gtest/samples/sample6_unittest.cc | 224 - pdal/test/gtest/samples/sample7_unittest.cc | 117 - pdal/test/gtest/samples/sample8_unittest.cc | 154 - pdal/test/gtest/samples/sample9_unittest.cc | 156 - pdal/test/gtest/scripts/common.py | 83 - pdal/test/gtest/scripts/fuse_gtest_files.py | 253 - .../test/gtest/scripts/gen_gtest_pred_impl.py | 730 - pdal/test/gtest/scripts/gtest-config.in | 274 - pdal/test/gtest/scripts/pump.py | 855 - pdal/test/gtest/scripts/release_docs.py | 158 - pdal/test/gtest/scripts/test/Makefile | 59 - pdal/test/gtest/scripts/upload.py | 1387 - pdal/test/gtest/scripts/upload_gtest.py | 78 - pdal/test/gtest/src/gtest-all.cc | 48 - pdal/test/gtest/src/gtest-death-test.cc | 1653 -- pdal/test/gtest/src/gtest-filepath.cc | 379 - pdal/test/gtest/src/gtest-internal-inl.h | 1211 - pdal/test/gtest/src/gtest-matchers.cc | 97 - pdal/test/gtest/src/gtest-port.cc | 1399 -- pdal/test/gtest/src/gtest-printers.cc | 442 - pdal/test/gtest/src/gtest-test-part.cc | 104 - pdal/test/gtest/src/gtest-typed-test.cc | 118 - pdal/test/gtest/src/gtest.cc | 6177 ----- pdal/test/gtest/src/gtest_main.cc | 47 - pdal/test/gtest/test/BUILD.bazel | 521 - .../googletest-break-on-failure-unittest.py | 208 - .../googletest-break-on-failure-unittest_.cc | 86 - .../test/googletest-catch-exceptions-test.py | 236 - .../test/googletest-catch-exceptions-test_.cc | 293 - pdal/test/gtest/test/googletest-color-test.py | 127 - .../test/gtest/test/googletest-color-test_.cc | 62 - .../gtest/test/googletest-death-test-test.cc | 1516 -- .../test/googletest-death-test_ex_test.cc | 92 - .../gtest/test/googletest-env-var-test.py | 117 - .../gtest/test/googletest-env-var-test_.cc | 122 - .../gtest/test/googletest-filepath-test.cc | 649 - .../gtest/test/googletest-filter-unittest.py | 639 - .../gtest/test/googletest-filter-unittest_.cc | 137 - .../test/googletest-json-outfiles-test.py | 191 - .../test/googletest-json-output-unittest.py | 778 - .../test/googletest-list-tests-unittest.py | 205 - .../test/googletest-list-tests-unittest_.cc | 156 - .../gtest/test/googletest-listener-test.cc | 518 - .../gtest/test/googletest-message-test.cc | 158 - .../gtest/test/googletest-options-test.cc | 216 - .../googletest-output-test-golden-lin.txt | 1140 - .../test/gtest/test/googletest-output-test.py | 346 - .../gtest/test/googletest-output-test_.cc | 1157 - ...oogletest-param-test-invalid-name1-test.py | 63 - ...ogletest-param-test-invalid-name1-test_.cc | 50 - ...oogletest-param-test-invalid-name2-test.py | 62 - ...ogletest-param-test-invalid-name2-test_.cc | 55 - .../gtest/test/googletest-param-test-test.cc | 1055 - .../gtest/test/googletest-param-test-test.h | 51 - .../gtest/test/googletest-param-test2-test.cc | 61 - pdal/test/gtest/test/googletest-port-test.cc | 1272 - .../gtest/test/googletest-printers-test.cc | 1620 -- .../gtest/test/googletest-shuffle-test.py | 323 - .../gtest/test/googletest-shuffle-test_.cc | 101 - .../gtest/test/googletest-test-part-test.cc | 230 - pdal/test/gtest/test/googletest-test2_test.cc | 61 - .../test/googletest-throw-on-failure-test.py | 168 - .../test/googletest-throw-on-failure-test_.cc | 71 - .../test/googletest-uninitialized-test.py | 67 - .../test/googletest-uninitialized-test_.cc | 42 - .../test/gtest/test/gtest-typed-test2_test.cc | 44 - pdal/test/gtest/test/gtest-typed-test_test.cc | 462 - pdal/test/gtest/test/gtest-typed-test_test.h | 65 - .../gtest/test/gtest-unittest-api_test.cc | 340 - pdal/test/gtest/test/gtest_all_test.cc | 46 - .../test/gtest_assert_by_exception_test.cc | 116 - .../test/gtest/test/gtest_environment_test.cc | 188 - pdal/test/gtest/test/gtest_help_test.py | 170 - pdal/test/gtest/test/gtest_help_test_.cc | 45 - pdal/test/gtest/test/gtest_json_test_utils.py | 60 - .../gtest/test/gtest_list_output_unittest.py | 141 - .../gtest/test/gtest_list_output_unittest_.cc | 51 - pdal/test/gtest/test/gtest_main_unittest.cc | 44 - .../test/gtest/test/gtest_no_test_unittest.cc | 54 - .../gtest/test/gtest_pred_impl_unittest.cc | 2427 -- .../gtest/test/gtest_premature_exit_test.cc | 126 - pdal/test/gtest/test/gtest_prod_test.cc | 56 - pdal/test/gtest/test/gtest_repeat_test.cc | 233 - ...test_skip_environment_check_output_test.py | 54 - .../gtest_skip_in_environment_setup_test.cc | 49 - pdal/test/gtest/test/gtest_skip_test.cc | 55 - .../test/gtest/test/gtest_sole_header_test.cc | 56 - pdal/test/gtest/test/gtest_stress_test.cc | 248 - .../gtest_test_macro_stack_footprint_test.cc | 89 - pdal/test/gtest/test/gtest_test_utils.py | 314 - pdal/test/gtest/test/gtest_testbridge_test.py | 63 - .../test/gtest/test/gtest_testbridge_test_.cc | 43 - .../test/gtest_throw_on_failure_ex_test.cc | 90 - pdal/test/gtest/test/gtest_unittest.cc | 7488 ------ .../gtest/test/gtest_xml_outfile1_test_.cc | 43 - .../gtest/test/gtest_xml_outfile2_test_.cc | 43 - .../gtest/test/gtest_xml_outfiles_test.py | 135 - .../gtest/test/gtest_xml_output_unittest.py | 389 - .../gtest/test/gtest_xml_output_unittest_.cc | 188 - pdal/test/gtest/test/gtest_xml_test_utils.py | 196 - pdal/test/gtest/test/production.cc | 35 - pdal/test/gtest/test/production.h | 54 - pdal/test/look.py | 7 - pdal/test/make-test-data.py | 10 - pdal/test/temp/README.txt | 1 - setup.py | 2 - test/test_pipeline.py | 50 +- test/test_pypy.py | 26 - 206 files changed, 123 insertions(+), 93817 deletions(-) create mode 100644 .github/environment.yml delete mode 100644 MANIFEST.in delete mode 100644 pdal/CMakeLists.txt delete mode 100644 pdal/filters/PythonFilter.cpp delete mode 100644 pdal/filters/PythonFilter.hpp delete mode 100644 pdal/io/NumpyReader.cpp delete mode 100644 pdal/io/NumpyReader.hpp delete mode 100644 pdal/nlohmann/json.hpp delete mode 100644 pdal/plang/Environment.cpp delete mode 100644 pdal/plang/Environment.hpp delete mode 100644 pdal/plang/Invocation.cpp delete mode 100644 pdal/plang/Invocation.hpp delete mode 100644 pdal/plang/Redirector.cpp delete mode 100644 pdal/plang/Redirector.hpp delete mode 100644 pdal/plang/Script.cpp delete mode 100644 pdal/plang/Script.hpp delete mode 100644 pdal/test/NumpyReaderTest.cpp delete mode 100644 pdal/test/PythonFilterTest.cpp delete mode 100644 pdal/test/Support.cpp delete mode 100644 pdal/test/Support.hpp delete mode 100644 pdal/test/TestConfig.hpp delete mode 100644 pdal/test/data/1.2-with-color.las delete mode 100644 pdal/test/data/1.2-with-color.npy delete mode 100644 pdal/test/data/autzen-utm.las delete mode 100644 pdal/test/data/autzen.las delete mode 100644 pdal/test/data/four-floats-be.bin delete mode 100644 pdal/test/data/four-floats.py delete mode 100644 pdal/test/data/logs/log_py.txt delete mode 100644 pdal/test/data/perlin.npy delete mode 100644 pdal/test/data/pipeline/crop_wkt_2d.json delete mode 100644 pdal/test/data/pipeline/crop_wkt_2d_classification.json delete mode 100644 pdal/test/data/pipeline/from-module.json delete mode 100644 pdal/test/data/pipeline/predicate-embed.json delete mode 100644 pdal/test/data/pipeline/predicate-keep-ground-and-unclass.json delete mode 100644 pdal/test/data/pipeline/predicate-keep-last-return.json delete mode 100644 pdal/test/data/pipeline/predicate-keep-specified-returns.json delete mode 100644 pdal/test/data/pipeline/programmable-update-y-dims.json delete mode 100644 pdal/test/data/pipeline/reproject.json delete mode 100644 pdal/test/data/test1.py delete mode 100644 pdal/test/data/threedim.npy delete mode 100644 pdal/test/data/threedim.py delete mode 100644 pdal/test/data/twodim.npy delete mode 100644 pdal/test/gtest/CMakeLists.txt delete mode 100644 pdal/test/gtest/CONTRIBUTORS delete mode 100644 pdal/test/gtest/LICENSE delete mode 100644 pdal/test/gtest/README.md delete mode 100644 pdal/test/gtest/cmake/Config.cmake.in delete mode 100644 pdal/test/gtest/cmake/gtest.pc.in delete mode 100644 pdal/test/gtest/cmake/gtest_main.pc.in delete mode 100644 pdal/test/gtest/cmake/internal_utils.cmake delete mode 100644 pdal/test/gtest/cmake/libgtest.la.in delete mode 100644 pdal/test/gtest/docs/advanced.md delete mode 100644 pdal/test/gtest/docs/faq.md delete mode 100644 pdal/test/gtest/docs/pkgconfig.md delete mode 100644 pdal/test/gtest/docs/primer.md delete mode 100644 pdal/test/gtest/docs/pump_manual.md delete mode 100644 pdal/test/gtest/docs/samples.md delete mode 100644 pdal/test/gtest/include/gtest/gtest-death-test.h delete mode 100644 pdal/test/gtest/include/gtest/gtest-matchers.h delete mode 100644 pdal/test/gtest/include/gtest/gtest-message.h delete mode 100644 pdal/test/gtest/include/gtest/gtest-param-test.h delete mode 100644 pdal/test/gtest/include/gtest/gtest-printers.h delete mode 100644 pdal/test/gtest/include/gtest/gtest-spi.h delete mode 100644 pdal/test/gtest/include/gtest/gtest-test-part.h delete mode 100644 pdal/test/gtest/include/gtest/gtest-typed-test.h delete mode 100644 pdal/test/gtest/include/gtest/gtest.h delete mode 100644 pdal/test/gtest/include/gtest/gtest_pred_impl.h delete mode 100644 pdal/test/gtest/include/gtest/gtest_prod.h delete mode 100644 pdal/test/gtest/include/gtest/internal/custom/README.md delete mode 100644 pdal/test/gtest/include/gtest/internal/custom/gtest-port.h delete mode 100644 pdal/test/gtest/include/gtest/internal/custom/gtest-printers.h delete mode 100644 pdal/test/gtest/include/gtest/internal/custom/gtest.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-death-test-internal.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-filepath.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-internal.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-param-util.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-port-arch.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-port.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-string.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-type-util.h delete mode 100644 pdal/test/gtest/include/gtest/internal/gtest-type-util.h.pump delete mode 100644 pdal/test/gtest/samples/prime_tables.h delete mode 100644 pdal/test/gtest/samples/sample1.cc delete mode 100644 pdal/test/gtest/samples/sample1.h delete mode 100644 pdal/test/gtest/samples/sample10_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample1_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample2.cc delete mode 100644 pdal/test/gtest/samples/sample2.h delete mode 100644 pdal/test/gtest/samples/sample2_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample3-inl.h delete mode 100644 pdal/test/gtest/samples/sample3_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample4.cc delete mode 100644 pdal/test/gtest/samples/sample4.h delete mode 100644 pdal/test/gtest/samples/sample4_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample5_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample6_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample7_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample8_unittest.cc delete mode 100644 pdal/test/gtest/samples/sample9_unittest.cc delete mode 100644 pdal/test/gtest/scripts/common.py delete mode 100755 pdal/test/gtest/scripts/fuse_gtest_files.py delete mode 100755 pdal/test/gtest/scripts/gen_gtest_pred_impl.py delete mode 100755 pdal/test/gtest/scripts/gtest-config.in delete mode 100755 pdal/test/gtest/scripts/pump.py delete mode 100755 pdal/test/gtest/scripts/release_docs.py delete mode 100644 pdal/test/gtest/scripts/test/Makefile delete mode 100755 pdal/test/gtest/scripts/upload.py delete mode 100755 pdal/test/gtest/scripts/upload_gtest.py delete mode 100644 pdal/test/gtest/src/gtest-all.cc delete mode 100644 pdal/test/gtest/src/gtest-death-test.cc delete mode 100644 pdal/test/gtest/src/gtest-filepath.cc delete mode 100644 pdal/test/gtest/src/gtest-internal-inl.h delete mode 100644 pdal/test/gtest/src/gtest-matchers.cc delete mode 100644 pdal/test/gtest/src/gtest-port.cc delete mode 100644 pdal/test/gtest/src/gtest-printers.cc delete mode 100644 pdal/test/gtest/src/gtest-test-part.cc delete mode 100644 pdal/test/gtest/src/gtest-typed-test.cc delete mode 100644 pdal/test/gtest/src/gtest.cc delete mode 100644 pdal/test/gtest/src/gtest_main.cc delete mode 100644 pdal/test/gtest/test/BUILD.bazel delete mode 100755 pdal/test/gtest/test/googletest-break-on-failure-unittest.py delete mode 100644 pdal/test/gtest/test/googletest-break-on-failure-unittest_.cc delete mode 100755 pdal/test/gtest/test/googletest-catch-exceptions-test.py delete mode 100644 pdal/test/gtest/test/googletest-catch-exceptions-test_.cc delete mode 100755 pdal/test/gtest/test/googletest-color-test.py delete mode 100644 pdal/test/gtest/test/googletest-color-test_.cc delete mode 100644 pdal/test/gtest/test/googletest-death-test-test.cc delete mode 100644 pdal/test/gtest/test/googletest-death-test_ex_test.cc delete mode 100755 pdal/test/gtest/test/googletest-env-var-test.py delete mode 100644 pdal/test/gtest/test/googletest-env-var-test_.cc delete mode 100644 pdal/test/gtest/test/googletest-filepath-test.cc delete mode 100755 pdal/test/gtest/test/googletest-filter-unittest.py delete mode 100644 pdal/test/gtest/test/googletest-filter-unittest_.cc delete mode 100644 pdal/test/gtest/test/googletest-json-outfiles-test.py delete mode 100644 pdal/test/gtest/test/googletest-json-output-unittest.py delete mode 100755 pdal/test/gtest/test/googletest-list-tests-unittest.py delete mode 100644 pdal/test/gtest/test/googletest-list-tests-unittest_.cc delete mode 100644 pdal/test/gtest/test/googletest-listener-test.cc delete mode 100644 pdal/test/gtest/test/googletest-message-test.cc delete mode 100644 pdal/test/gtest/test/googletest-options-test.cc delete mode 100644 pdal/test/gtest/test/googletest-output-test-golden-lin.txt delete mode 100755 pdal/test/gtest/test/googletest-output-test.py delete mode 100644 pdal/test/gtest/test/googletest-output-test_.cc delete mode 100644 pdal/test/gtest/test/googletest-param-test-invalid-name1-test.py delete mode 100644 pdal/test/gtest/test/googletest-param-test-invalid-name1-test_.cc delete mode 100644 pdal/test/gtest/test/googletest-param-test-invalid-name2-test.py delete mode 100644 pdal/test/gtest/test/googletest-param-test-invalid-name2-test_.cc delete mode 100644 pdal/test/gtest/test/googletest-param-test-test.cc delete mode 100644 pdal/test/gtest/test/googletest-param-test-test.h delete mode 100644 pdal/test/gtest/test/googletest-param-test2-test.cc delete mode 100644 pdal/test/gtest/test/googletest-port-test.cc delete mode 100644 pdal/test/gtest/test/googletest-printers-test.cc delete mode 100755 pdal/test/gtest/test/googletest-shuffle-test.py delete mode 100644 pdal/test/gtest/test/googletest-shuffle-test_.cc delete mode 100644 pdal/test/gtest/test/googletest-test-part-test.cc delete mode 100644 pdal/test/gtest/test/googletest-test2_test.cc delete mode 100755 pdal/test/gtest/test/googletest-throw-on-failure-test.py delete mode 100644 pdal/test/gtest/test/googletest-throw-on-failure-test_.cc delete mode 100755 pdal/test/gtest/test/googletest-uninitialized-test.py delete mode 100644 pdal/test/gtest/test/googletest-uninitialized-test_.cc delete mode 100644 pdal/test/gtest/test/gtest-typed-test2_test.cc delete mode 100644 pdal/test/gtest/test/gtest-typed-test_test.cc delete mode 100644 pdal/test/gtest/test/gtest-typed-test_test.h delete mode 100644 pdal/test/gtest/test/gtest-unittest-api_test.cc delete mode 100644 pdal/test/gtest/test/gtest_all_test.cc delete mode 100644 pdal/test/gtest/test/gtest_assert_by_exception_test.cc delete mode 100644 pdal/test/gtest/test/gtest_environment_test.cc delete mode 100755 pdal/test/gtest/test/gtest_help_test.py delete mode 100644 pdal/test/gtest/test/gtest_help_test_.cc delete mode 100644 pdal/test/gtest/test/gtest_json_test_utils.py delete mode 100644 pdal/test/gtest/test/gtest_list_output_unittest.py delete mode 100644 pdal/test/gtest/test/gtest_list_output_unittest_.cc delete mode 100644 pdal/test/gtest/test/gtest_main_unittest.cc delete mode 100644 pdal/test/gtest/test/gtest_no_test_unittest.cc delete mode 100644 pdal/test/gtest/test/gtest_pred_impl_unittest.cc delete mode 100644 pdal/test/gtest/test/gtest_premature_exit_test.cc delete mode 100644 pdal/test/gtest/test/gtest_prod_test.cc delete mode 100644 pdal/test/gtest/test/gtest_repeat_test.cc delete mode 100755 pdal/test/gtest/test/gtest_skip_environment_check_output_test.py delete mode 100644 pdal/test/gtest/test/gtest_skip_in_environment_setup_test.cc delete mode 100644 pdal/test/gtest/test/gtest_skip_test.cc delete mode 100644 pdal/test/gtest/test/gtest_sole_header_test.cc delete mode 100644 pdal/test/gtest/test/gtest_stress_test.cc delete mode 100644 pdal/test/gtest/test/gtest_test_macro_stack_footprint_test.cc delete mode 100755 pdal/test/gtest/test/gtest_test_utils.py delete mode 100755 pdal/test/gtest/test/gtest_testbridge_test.py delete mode 100644 pdal/test/gtest/test/gtest_testbridge_test_.cc delete mode 100644 pdal/test/gtest/test/gtest_throw_on_failure_ex_test.cc delete mode 100644 pdal/test/gtest/test/gtest_unittest.cc delete mode 100644 pdal/test/gtest/test/gtest_xml_outfile1_test_.cc delete mode 100644 pdal/test/gtest/test/gtest_xml_outfile2_test_.cc delete mode 100755 pdal/test/gtest/test/gtest_xml_outfiles_test.py delete mode 100755 pdal/test/gtest/test/gtest_xml_output_unittest.py delete mode 100644 pdal/test/gtest/test/gtest_xml_output_unittest_.cc delete mode 100755 pdal/test/gtest/test/gtest_xml_test_utils.py delete mode 100644 pdal/test/gtest/test/production.cc delete mode 100644 pdal/test/gtest/test/production.h delete mode 100644 pdal/test/look.py delete mode 100644 pdal/test/make-test-data.py delete mode 100644 pdal/test/temp/README.txt delete mode 100644 test/test_pypy.py diff --git a/.github/environment.yml b/.github/environment.yml new file mode 100644 index 00000000..d757c9d2 --- /dev/null +++ b/.github/environment.yml @@ -0,0 +1,10 @@ +name: test +channels: + - conda-forge +dependencies: + - scikit-build + - numpy + - pybind11 + - pdal + - pytest + - meshio diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f9ac47fb..7e982ca1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,121 +10,83 @@ on: release: types: - published -jobs: - unix: - name: ${{ matrix.os }} - ${{ matrix.python-version }} +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + build: + name: Build ${{ matrix.os }} py${{ matrix.python-version }} runs-on: ${{ matrix.os }} + strategy: fail-fast: true matrix: - os: ['ubuntu-latest', 'macos-latest' ] - python-version: ['3.7','3.8', '3.9'] + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + python-version: ['3.7', '3.8', '3.9'] steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: ${{ matrix.python-version }} - mamba-version: "*" - activate-environment: test - channels: conda-forge - auto-activate-base: false - - - name: Linux dependencies - if: matrix.os == 'ubuntu-latest' - shell: bash -l {0} - run: sudo apt-get install gcc-multilib -y - - - name: Dependencies - shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} pybind11 pdal pytest meshio - - - name: Install - shell: bash -l {0} - run: | - python setup.py develop -- -G "Ninja" -DWITH_TESTS=ON - pdal --drivers + - name: Check out python-pdal + uses: actions/checkout@v2 - - name: Test - shell: bash -l {0} - run: | - export PDAL_DRIVER_PATH=$(python -c "import skbuild; print(skbuild.constants.SKBUILD_DIR())")/cmake-build - $PDAL_DRIVER_PATH/pdal_filters_python_test - $PDAL_DRIVER_PATH/pdal_io_numpy_test - py.test -v test/ - - windows: - name: ${{ matrix.os }} - ${{ matrix.python-version }} - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - os: ['windows-latest'] - python-version: ['3.7','3.8', '3.9'] + - name: Check out python-pdal-plugins + uses: actions/checkout@v2 + with: + repository: PDAL/python-plugins + path: ./plugins - steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 + - name: Setup micromamba + uses: mamba-org/provision-with-micromamba@main with: - channels: conda-forge - python-version: ${{ matrix.python-version }} + environment-file: .github/environment.yml + extra-specs: | + python=${{ matrix.python-version }} - - name: Dependencies - shell: cmd /C CALL "{0}" - run: | - call conda activate test - conda install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} pybind11 pdal pytest meshio + - name: Install python-pdal + run: pip install -e . - - name: Install - shell: cmd /C CALL "{0}" - run: | - call conda activate test - python setup.py develop -- -G "Ninja" -DWITH_TESTS=ON - pdal --drivers + - name: Install python-pdal-plugins + working-directory: ./plugins + run: pip install -e . - name: Test - shell: cmd /C CALL "{0}" run: | - call conda activate test - set PYTHONHOME=%CONDA_PREFIX% - for /f %%i in ('python -c "import skbuild; print(skbuild.constants.SKBUILD_DIR())"') do set PDAL_DRIVER_PATH=%%i\cmake-build - %PDAL_DRIVER_PATH%\pdal_filters_python_test.exe - %PDAL_DRIVER_PATH%\pdal_io_numpy_test.exe + export PDAL_DRIVER_PATH=$(python -c "import os, skbuild; print(os.path.join('plugins', skbuild.constants.SKBUILD_DIR(), 'cmake-build'))") + pdal --drivers py.test -v test/ - dist: - name: Distribution - needs: [windows, unix] + - name: Build wheel distribution + run: python setup.py --skip-cmake bdist_wheel - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - os: ['ubuntu-latest'] - python-version: ['3.8'] + - name: Build source distribution + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.8' + run: python setup.py sdist - steps: - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 + - name: Upload distribution(s) + uses: actions/upload-artifact@v2 with: - channels: conda-forge - python-version: ${{ matrix.python-version }} - mamba-version: "*" + name: ${{ matrix.os }}-py${{ matrix.python-version }} + path: ./dist/* - - name: Dependencies - shell: bash -l {0} - run: mamba install --yes --quiet -c conda-forge scikit-build numpy python=${{ matrix.python-version }} pybind11 pdal + publish: + runs-on: ubuntu-latest + needs: [build] + + steps: + - name: Download distribution(s) + uses: actions/download-artifact@v2 + with: + path: ./artifacts - - name: sdist - shell: bash -l {0} + - name: Move artifacts to dist run: | - python setup.py sdist - ls dist + mkdir dist + find ./artifacts -type f -exec mv {} ./dist \; + tree ./dist - - uses: pypa/gh-action-pypi-publish@master - name: Publish package + - name: Publish package + uses: pypa/gh-action-pypi-publish@master if: github.event_name == 'release' && github.event.action == 'published' with: user: __token__ diff --git a/CMakeLists.txt b/CMakeLists.txt index f0c2ecdc..98e94753 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,6 @@ project(PDAL) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -option(WITH_TESTS "Enable tests" OFF) set(CMAKE_BUILD_TYPE "Release") # Python-finding settings @@ -16,249 +15,16 @@ find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) # find PDAL. Require 2.1+ find_package(PDAL 2.1 REQUIRED) -add_subdirectory(pdal) - -# Taken and adapted from PDAL's cmake macros.cmake - -function(pdal_python_target_compile_settings target) - set_property(TARGET ${target} PROPERTY CXX_STANDARD 11) - set_property(TARGET ${target} PROPERTY CXX_STANDARD_REQUIRED TRUE) - target_compile_definitions(${target} PRIVATE - -DWIN32_LEAN_AND_MEAN) - if (MSVC) - # check for MSVC 8+ - if (NOT (MSVC_VERSION VERSION_LESS 1400)) - target_compile_definitions(${target} PRIVATE - -D_CRT_SECURE_NO_DEPRECATE - -D_CRT_SECURE_NO_WARNINGS - -D_CRT_NONSTDC_NO_WARNING - -D_SCL_SECURE_NO_WARNINGS - ) - target_compile_options(${target} PRIVATE - # Yes, we don't understand GCC pragmas - /wd4068 - # Nitro makes use of Exception Specifications, which results in - # numerous warnings when compiling in MSVC. We will ignore - # them for now. - /wd4290 - /wd4800 - # Windows warns about integer narrowing like crazy and it's - # annoying. In most cases the programmer knows what they're - # doing. A good static analysis tool would be better than - # turning this warning off. - /wd4267 - # Annoying warning about function hiding with virtual - # inheritance. - /wd4250 - # some templates don't return -# /wd4716 - # unwind semantics -# /wd4530 - # Standard C++-type exception handling. - /EHsc - ) - endif() - - endif() -endfunction() - - -############################################################################### -# Add a plugin target. -# _name The plugin name. -# ARGN : -# FILES the source files for the plugin -# LINK_WITH link plugin with libraries -# INCLUDES header directories -# -# The "generate_dimension_hpp" ensures that Dimension.hpp is built before -# attempting to build anything else in the "library". -# -# NOTE: _name is the name of a variable that will hold the plugin name -# when the macro completes -macro(PDAL_PYTHON_ADD_PLUGIN _name _type _shortname) - set(options) - set(oneValueArgs) - set(multiValueArgs FILES LINK_WITH INCLUDES SYSTEM_INCLUDES COMPILE_OPTIONS) - cmake_parse_arguments(PDAL_PYTHON_ADD_PLUGIN "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN}) - if(WIN32) - set(WINSOCK_LIBRARY ws2_32) - set(${_name} "libpdal_plugin_${_type}_${_shortname}") - else() - set(${_name} "pdal_plugin_${_type}_${_shortname}") - endif() - - - add_library(${${_name}} SHARED ${PDAL_PYTHON_ADD_PLUGIN_FILES}) - pdal_python_target_compile_settings(${${_name}}) - target_include_directories(${${_name}} PRIVATE - ${PROJECT_BINARY_DIR}/include - ${PDAL_INCLUDE_DIR} - ${PDAL_PYTHON_ADD_PLUGIN_INCLUDES} - ) - target_link_options(${${_name}} BEFORE PRIVATE ${PDAL_PYTHON_ADD_PLUGIN_COMPILE_OPTIONS}) - target_compile_definitions(${${_name}} PRIVATE - PDAL_PYTHON_LIBRARY="${PYTHON_LIBRARY}" PDAL_DLL_EXPORT) - target_compile_definitions(${${_name}} PRIVATE PDAL_DLL_EXPORT) - if (PDAL_PYTHON_ADD_PLUGIN_SYSTEM_INCLUDES) - target_include_directories(${${_name}} SYSTEM PRIVATE - ${PDAL_PYTHON_ADD_PLUGIN_SYSTEM_INCLUDES}) - endif() - target_link_libraries(${${_name}} - PRIVATE - ${PDAL_PYTHON_ADD_PLUGIN_LINK_WITH} - ${WINSOCK_LIBRARY} - ) - - message(STATUS "PROJECT_NAME: ${PROJECT_NAME}") - install(TARGETS ${${_name}} - LIBRARY DESTINATION "pdal" - ) - if (APPLE) - set_target_properties(${${_name}} PROPERTIES - INSTALL_NAME_DIR "@rpath") - endif() -endmacro(PDAL_PYTHON_ADD_PLUGIN) - - -macro(PDAL_PYTHON_ADD_TEST _name) - set(options) - set(oneValueArgs) - set(multiValueArgs FILES LINK_WITH INCLUDES SYSTEM_INCLUDES) - cmake_parse_arguments(PDAL_PYTHON_ADD_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (WIN32) - set(WINSOCK_LIBRARY ws2_32) - endif() - add_executable(${_name} ${PDAL_PYTHON_ADD_TEST_FILES}) - - pdal_python_target_compile_settings(${_name}) - target_include_directories(${_name} PRIVATE - ${PDAL_PYTHON_ADD_TEST_INCLUDES}) - if (PDAL_PYTHON_ADD_TEST_SYSTEM_INCLUDES) - target_include_directories(${_name} SYSTEM PRIVATE - ${PDAL_PYTHON_ADD_TEST_SYSTEM_INCLUDES}) - endif() - set_property(TARGET ${_name} PROPERTY FOLDER "Tests") - target_link_libraries(${_name} - PRIVATE - ${PDAL_PYTHON_ADD_TEST_LINK_WITH} - gtest - ${WINSOCK_LIBRARY} - ) - target_compile_definitions(${_name} PRIVATE - PDAL_PYTHON_LIBRARY="${PYTHON_LIBRARY}") - add_test(NAME ${_name} - COMMAND - "${PROJECT_BINARY_DIR}/bin/${_name}" - WORKING_DIRECTORY - "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/..") - # Ensure plugins are loaded from build dir - # https://github.com/PDAL/PDAL/issues/840 -# if (WIN32) -# set_property(TEST ${_name} PROPERTY ENVIRONMENT -# "PDAL_DRIVER_PATH=${PROJECT_BINARY_DIR}/bin") -# else() -# set_property(TEST ${_name} PROPERTY ENVIRONMENT -# "PDAL_DRIVER_PATH=${PROJECT_BINARY_DIR}/lib") -# endif() -endmacro(PDAL_PYTHON_ADD_TEST) - -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - # For newer versions of python (3.8+), C extensions don't link against - # libpython and instead get symbol definitions from the python interpreter - # executable. PDAL plugins need to link against libpython, but if a plugin - # is loaded inside a python process, it must resolve symbols from the python - # executable instead of libpython. Using flat namespace allows that. - set(PYTHON_LINK_LIBRARY ${PYTHON_LINK_LIBRARY} -Wl,-flat_namespace) -endif() - -PDAL_PYTHON_ADD_PLUGIN(numpy_reader reader numpy - FILES - ./pdal/io/NumpyReader.cpp - ./pdal/io/NumpyReader.hpp - ./pdal/plang/Invocation.cpp - ./pdal/plang/Environment.cpp - ./pdal/plang/Redirector.cpp - ./pdal/plang/Script.cpp - LINK_WITH - ${PDAL_LIBRARIES} - ${Python3_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - COMPILE_OPTIONS - ${PYTHON_LINK_LIBRARY} - ) - -PDAL_PYTHON_ADD_PLUGIN(python_filter filter python - FILES - ./pdal/filters/PythonFilter.cpp - ./pdal/filters/PythonFilter.hpp - ./pdal/plang/Invocation.cpp - ./pdal/plang/Environment.cpp - ./pdal/plang/Redirector.cpp - ./pdal/plang/Script.cpp - LINK_WITH - ${PDAL_LIBRARIES} - ${Python3_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - COMPILE_OPTIONS - ${PYTHON_LINK_LIBRARY} - ) - - -if (WITH_TESTS) - enable_testing() - set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - add_subdirectory(pdal/test/gtest) - enable_testing() - include_directories(pdal/test/gtest/include .. ${CMAKE_CURRENT_BINARY_DIR}) - - - - - PDAL_PYTHON_ADD_TEST(pdal_io_numpy_test - FILES - ./pdal/test/NumpyReaderTest.cpp - ./pdal/test/Support.cpp - ./pdal/plang/Invocation.cpp - ./pdal/plang/Environment.cpp - ./pdal/plang/Redirector.cpp - ./pdal/plang/Script.cpp - LINK_WITH - ${numpy_reader} - ${Python3_LIBRARIES} - ${PDAL_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - ) - - PDAL_PYTHON_ADD_TEST(pdal_filters_python_test - FILES - ./pdal/test/PythonFilterTest.cpp - ./pdal/test/Support.cpp - ./pdal/plang/Invocation.cpp - ./pdal/plang/Environment.cpp - ./pdal/plang/Redirector.cpp - ./pdal/plang/Script.cpp - LINK_WITH - ${python_filter} - ${Python3_LIBRARIES} - ${PDAL_LIBRARIES} - ${CMAKE_DL_LIBS} - SYSTEM_INCLUDES - ${PDAL_INCLUDE_DIRS} - ${Python3_INCLUDE_DIRS} - ${Python3_NumPy_INCLUDE_DIRS} - ) -endif (WITH_TESTS) +# find PyBind11 +find_package(pybind11 REQUIRED) + +set(extension "libpdalpython") +pybind11_add_module(${extension} MODULE + pdal/PyArray.cpp + pdal/PyPipeline.cpp + pdal/StreamableExecutor.cpp + pdal/libpdalpython.cpp +) +target_include_directories(${extension} PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) +target_link_libraries(${extension} PRIVATE ${PDAL_LIBRARIES}) +install(TARGETS ${extension} LIBRARY DESTINATION "pdal") diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index b7a2acb1..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,8 +0,0 @@ -exclude MANIFEST.in -include LICENSE.txt CHANGES.txt README.rst pyproject.toml CMakeLists.txt pdal/CMakeLists.txt -include test/*py -include test/data/* - -recursive-include pdal/test * -recursive-include pdal *.hpp -recursive-include pdal *.cpp diff --git a/README.rst b/README.rst index 37ca51b9..b4ed7db5 100644 --- a/README.rst +++ b/README.rst @@ -3,13 +3,8 @@ PDAL ================================================================================ PDAL Python support allows you to process data with PDAL into `Numpy`_ arrays. -It supports embedding Python in PDAL pipelines with the `readers.numpy -`__ and `filters.python -`__ stages, and it provides a PDAL -extension module to control Python interaction with PDAL. - -Additionally, you can use it to fetch `schema`_ and `metadata`_ from -PDAL operations. +It provides a PDAL extension module to control Python interaction with PDAL. +Additionally, you can use it to fetch `schema`_ and `metadata`_ from PDAL operations. Installation -------------------------------------------------------------------------------- diff --git a/pdal/CMakeLists.txt b/pdal/CMakeLists.txt deleted file mode 100644 index 8886a465..00000000 --- a/pdal/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -find_package(pybind11 REQUIRED) -set(extension "libpdalpython") -pybind11_add_module(${extension} MODULE - PyArray.cpp PyPipeline.cpp StreamableExecutor.cpp libpdalpython.cpp -) -target_include_directories(${extension} PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) -target_link_libraries(${extension} PRIVATE ${PDAL_LIBRARIES}) -install(TARGETS ${extension} LIBRARY DESTINATION "pdal") diff --git a/pdal/filters/PythonFilter.cpp b/pdal/filters/PythonFilter.cpp deleted file mode 100644 index 6b8f5399..00000000 --- a/pdal/filters/PythonFilter.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/****************************************************************************** -* Copyright (c) 2011, Michael P. Gerlek (mpg@flaxen.com) -* -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following -* conditions are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided -* with the distribution. -* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the -* names of its contributors may be used to endorse or promote -* products derived from this software without specific prior -* written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -* OF SUCH DAMAGE. -****************************************************************************/ - -#include "PythonFilter.hpp" - -#include "../nlohmann/json.hpp" - -#include -#include -#include -#include - -namespace pdal -{ - -static PluginInfo const s_info -{ - "filters.python", - "Manipulate data using inline Python", - "http://pdal.io/stages/filters.python.html" -}; - -CREATE_SHARED_STAGE(PythonFilter, s_info) - -struct PythonFilter::Args -{ - std::string m_module; - std::string m_function; - std::string m_source; - std::string m_scriptFile; - StringList m_addDimensions; - NL::json m_pdalargs; -}; - -PythonFilter::PythonFilter() : - m_script(nullptr), m_pythonMethod(nullptr), m_args(new Args) -{} - - -PythonFilter::~PythonFilter() -{} - - -std::string PythonFilter::getName() const -{ - return s_info.name; -} - - -void PythonFilter::addArgs(ProgramArgs& args) -{ - args.add("module", "Python module containing the function to run", - m_args->m_module).setPositional(); - args.add("function", "Function to call", - m_args->m_function).setPositional(); - args.add("source", "Python script to run", m_args->m_source); - args.add("script", "File containing script to run", m_args->m_scriptFile); - args.add("add_dimension", "Dimensions to add", m_args->m_addDimensions); - args.add("pdalargs", "Dictionary to add to module globals when " - "calling function", m_args->m_pdalargs); -} - - -void PythonFilter::addDimensions(PointLayoutPtr layout) -{ - for (const std::string& s : m_args->m_addDimensions) - { - StringList spec = Utils::split(s, '='); - Utils::trim(spec[0]); - if (spec.size() == 2) - { - Utils::trim(spec[1]); - auto type = pdal::Dimension::type(spec[1]); - if (type == pdal::Dimension::Type::None) - throwError("Invalid dimension type specified '" + spec[1] + - "'. See documentation for valid dimension types"); - layout->registerOrAssignDim(spec[0], type); - } - else if (spec.size() == 1) - layout->registerOrAssignDim(s, pdal::Dimension::Type::Double); - else - throwError("Invalid dimension specified '" + s + - "'. Need or =."); - } -} - - -void PythonFilter::prepared(PointTableRef table) -{ - if (m_args->m_source.size() && m_args->m_scriptFile.size()) - throwError("Can't set both 'source' and 'script' options."); - if (!m_args->m_source.size() && !m_args->m_scriptFile.size()) - throwError("Must set one of 'source' and 'script' options."); -} - - -void PythonFilter::ready(PointTableRef table) -{ - if (m_args->m_source.empty()) - m_args->m_source = FileUtils::readFileIntoString(m_args->m_scriptFile); - std::ostream *out = log()->getLogStream(); - plang::EnvironmentPtr env = plang::Environment::get(); - env->set_stdout(out); - m_script.reset(new plang::Script(m_args->m_source, m_args->m_module, - m_args->m_function)); - m_pythonMethod.reset(new plang::Invocation(*m_script, table.metadata(), - m_args->m_pdalargs.dump(1))); -} - - -PointViewSet PythonFilter::run(PointViewPtr view) -{ - log()->get(LogLevel::Debug5) << "filters.python " << *m_script << - " processing " << view->size() << " points." << std::endl; - m_pythonMethod->execute(view, getMetadata()); - - PointViewSet viewSet; - viewSet.insert(view); - return viewSet; -} - - -void PythonFilter::done(PointTableRef table) -{ - static_cast(plang::Environment::get())->reset_stdout(); -} - -} // namespace pdal diff --git a/pdal/filters/PythonFilter.hpp b/pdal/filters/PythonFilter.hpp deleted file mode 100644 index 19f3d036..00000000 --- a/pdal/filters/PythonFilter.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/****************************************************************************** -* Copyright (c) 2011, Michael P. Gerlek (mpg@flaxen.com) -* -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following -* conditions are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided -* with the distribution. -* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the -* names of its contributors may be used to endorse or promote -* products derived from this software without specific prior -* written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -* OF SUCH DAMAGE. -****************************************************************************/ - -#pragma once - -#include -#include - -#include "../plang/Invocation.hpp" - - -namespace pdal -{ - -class PDAL_DLL PythonFilter : public Filter -{ -public: - PythonFilter(); - virtual ~PythonFilter(); - - std::string getName() const; - -private: - PythonFilter& operator=(const PythonFilter&) = delete; - PythonFilter(const PythonFilter&) = delete; - - virtual void addArgs(ProgramArgs& args); - virtual void addDimensions(PointLayoutPtr layout); - virtual void prepared(PointTableRef table); - virtual void ready(PointTableRef table); - virtual PointViewSet run(PointViewPtr view); - virtual void done(PointTableRef table); - - std::unique_ptr m_script; - std::unique_ptr m_pythonMethod; - - struct Args; - std::unique_ptr m_args; -}; - -} // namespace pdal diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp deleted file mode 100644 index 8983340e..00000000 --- a/pdal/io/NumpyReader.cpp +++ /dev/null @@ -1,684 +0,0 @@ -/****************************************************************************** -* Copyright (c) 2018, Howard Butler, howard@hobu.co -* -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following -* conditions are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided -* with the distribution. -* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the -* names of its contributors may be used to endorse or promote -* products derived from this software without specific prior -* written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -* OF SUCH DAMAGE. -****************************************************************************/ - -#include "NumpyReader.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../plang/Environment.hpp" - - -std::string toString(PyObject *pname) -{ - std::stringstream mssg; - PyObject* r = PyObject_Str(pname); - if (!r) - throw pdal::pdal_error("couldn't make string representation value"); -#if PY_MAJOR_VERSION >= 3 - Py_ssize_t size; - const char *d = PyUnicode_AsUTF8AndSize(r, &size); -#else - const char *d = PyString_AsString(r); -#endif - mssg << d; - return mssg.str(); -} - - -namespace pdal -{ - -static PluginInfo const s_info -{ - "readers.numpy", - "Read data from .npy files.", - "" -}; - -struct NumpyReader::Args -{ - std::string module; - std::string function; - std::string source; - std::string fargs; -}; - -CREATE_SHARED_STAGE(NumpyReader, s_info) - -std::string NumpyReader::getName() const { return s_info.name; } - -std::ostream& operator << (std::ostream& out, - const NumpyReader::Order& order) -{ - switch (order) - { - case NumpyReader::Order::Row: - out << "row"; - break; - case NumpyReader::Order::Column: - out << "column"; - break; - } - return out; -} - - -std::istream& operator >> (std::istream& in, NumpyReader::Order& order) -{ - std::string s; - in >> s; - - s = Utils::tolower(s); - if (s == "row") - order = NumpyReader::Order::Row; - else if (s == "column") - order = NumpyReader::Order::Column; - else - in.setstate(std::ios_base::failbit); - return in; -} - -NumpyReader::NumpyReader() - : m_array(nullptr) - , m_args(new NumpyReader::Args) -{} - -NumpyReader::~NumpyReader() -{} - -void NumpyReader::setArray(PyObject* array) -{ - plang::Environment::get(); - if (!PyArray_Check(array)) - throw pdal::pdal_error("object provided to setArray is not a python numpy array!"); - - m_array = (PyArrayObject*)array; - Py_XINCREF(m_array); -} - -PyArrayObject* load_npy_file(std::string const& filename) -{ - - PyObject *py_filename = PyUnicode_FromString(filename.c_str()); - PyObject *numpy_module = PyImport_ImportModule("numpy"); - if (!numpy_module) - throw pdal::pdal_error(plang::getTraceback()); - - PyObject *numpy_mod_dict = PyModule_GetDict(numpy_module); - if (!numpy_mod_dict) - throw pdal::pdal_error(plang::getTraceback()); - - PyObject *loads_func = PyDict_GetItemString(numpy_mod_dict, "load"); - if (!loads_func) - throw pdal::pdal_error(plang::getTraceback()); - - PyObject *numpy_args = PyTuple_New(1); - if (!numpy_args) - throw pdal::pdal_error(plang::getTraceback()); - - if (PyTuple_SetItem(numpy_args, 0, py_filename)) - throw pdal::pdal_error(plang::getTraceback()); - - PyObject* array = PyObject_CallObject(loads_func, numpy_args); - if (!array) - throw pdal::pdal_error(plang::getTraceback()); - - return reinterpret_cast(array); -} - -PyArrayObject* load_npy_script(std::string const& source, - std::string const& module, - std::string const& function, - std::string const& fargs) -{ - - MetadataNode m; - plang::Script script(source, module, function); - plang::Invocation method(script, m, fargs); - - StringList args = Utils::split(fargs,','); - - PyObject *scriptArgs = PyTuple_New(args.size()); - for(size_t i=0; i < args.size(); ++i) - { - PyObject* arg = PyUnicode_FromString(args[i].c_str()); - if (!arg) - throw pdal_error(plang::getTraceback()); - PyTuple_SetItem(scriptArgs, i, arg); - - } - - PyObject *array = PyObject_CallObject(method.m_function, scriptArgs); - if (!array) - throw pdal_error(plang::getTraceback()); - - Py_XDECREF(scriptArgs); - - return reinterpret_cast(array); -} - -void NumpyReader::initialize() -{ - plang::Environment::get(); - m_numPoints = 0; - m_chunkCount = 0; - m_ndims = 0; - - m_iter = NULL; - - p_data = NULL; - m_dataptr = NULL; - m_strideptr = NULL; - m_innersizeptr = NULL; - m_dtype = NULL; - - if (m_args->function.size() ) - { - // Invoke the script and use the returned - // numpy array - // - m_args->source = pdal::FileUtils::readFileIntoString(m_filename); - m_array = load_npy_script(m_args->source, - m_args->module, - m_args->function, - m_args->fargs); - if (!PyArray_Check(m_array)) - { - std::stringstream errMsg; - errMsg << "Object returned from function '" - << m_args->function << - "' in '" << m_filename << - "' is not a Numpy array"; - throw pdal::pdal_error(errMsg.str()); - } - } - else if (m_filename.size()) - { - m_array = load_npy_file(m_filename); - if (!PyArray_Check(m_array)) - throw pdal::pdal_error("Object in file '" + m_filename + - "' is not a Numpy array"); - - } -} - - -void NumpyReader::wakeUpNumpyArray() -{ - // TODO pivot whether we are a 1d, 2d, or named arrays - - if (PyArray_SIZE(m_array) == 0) - throw pdal::pdal_error("Array cannot be empty!"); - - m_iter = NpyIter_New(m_array, - NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY| NPY_ITER_REFS_OK, - NPY_KEEPORDER, NPY_NO_CASTING, - NULL); - if (!m_iter) - { - std::ostringstream oss; - - oss << "Unable to create iterator from array in '" - << m_filename + "' with traceback: '" - << plang::getTraceback() <<"'"; - throw pdal::pdal_error(oss.str()); - } - - char* itererr; - m_iternext = NpyIter_GetIterNext(m_iter, &itererr); - if (!m_iternext) - { - NpyIter_Deallocate(m_iter); - throw pdal::pdal_error(itererr); - } - - m_dtype = PyArray_DTYPE(m_array); - if (!m_dtype) - throw pdal_error(plang::getTraceback()); - - m_ndims = PyArray_NDIM(m_array); - m_shape = PyArray_SHAPE(m_array); - if (!m_shape) - throw pdal_error(plang::getTraceback()); - - npy_intp* shape = PyArray_SHAPE(m_array); - if (!shape) - throw pdal_error(plang::getTraceback()); - m_numPoints = 1; - for (int i = 0; i < m_ndims; ++i) - m_numPoints *= m_shape[i]; - - // If the order arg wasn't set, set order based on the internal order of - // the array. - if (!m_orderArg->set()) - { - int flags = PyArray_FLAGS(m_array); - if (flags & NPY_ARRAY_F_CONTIGUOUS) - m_order = Order::Column; - else - m_order = Order::Row; - } -} - - -void NumpyReader::addArgs(ProgramArgs& args) -{ - args.add("dimension", "In an unstructured array, the dimension name to " - "map to values.", m_defaultDimension, "Intensity"); - m_orderArg = &args.add("order", "Order of dimension interpretation " - "of the array. Either 'row'-major (C) or 'column'-major (Fortran)", - m_order); - - args.add("module", "Python module containing the function to run", - m_args->module); - args.add("function", "Function nameto call", - m_args->function); - args.add("fargs", "Args to call function with ", m_args->fargs); - -} - - -// Try to map dimension names to existing PDAL dimension names by -// checking them with certain characters removed. -Dimension::Id NumpyReader::registerDim(PointLayoutPtr layout, - const std::string& name, Dimension::Type pdalType) -{ - Dimension::Id id; - - auto registerName = [&layout, &pdalType, &id](std::string name, char elim) - { - if (elim != '\0') - Utils::remove(name, elim); - Dimension::Id tempId = Dimension::id(name); - if (tempId != Dimension::Id::Unknown) - { - id = tempId; - layout->registerDim(id, pdalType); - return true; - } - return false; - }; - - // Try registering the name in various ways. If that doesn't work, - // just punt and use the name as is. - if (!registerName(name, '\0') && !registerName(name, '-') && - !registerName(name, ' ') && !registerName(name, '_')) - id = layout->registerOrAssignDim(name, pdalType); - return id; -} - -namespace -{ - - -Dimension::Type getType(PyArray_Descr *dtype, const std::string& name) -{ - if (!dtype) - throw pdal_error("Can't fetch data type for numpy field."); - - Dimension::Type pdalType = - plang::Environment::getPDALDataType(dtype->type_num); - if (pdalType == Dimension::Type::None) - { - std::ostringstream oss; - oss << "Unable to map dimension '" << name << "' because its " - "type '" << dtype->type_num <<"' is not mappable to PDAL"; - throw pdal_error(oss.str()); - } - return pdalType; -} - -} // unnamed namespace - - -void NumpyReader::createFields(PointLayoutPtr layout) -{ - Dimension::Id id; - Dimension::Type type; - int offset; - - m_numFields = 0; - if (m_dtype->fields != Py_None) - m_numFields = static_cast(PyDict_Size(m_dtype->fields)); - - // Array isn't structured - just a bunch of data. - if (m_numFields <= 0) - { - type = getType(m_dtype, m_defaultDimension); - id = registerDim(layout, m_defaultDimension, type); - m_fields.push_back({id, type, 0}); - } - else - { - PyObject* names_dict = m_dtype->fields; - PyObject* names = PyDict_Keys(names_dict); - PyObject* values = PyDict_Values(names_dict); - if (!names || !values) - throw pdal_error("Bad field specification for numpy array layout."); - - for (int i = 0; i < m_numFields; ++i) - { - std::string name = toString(PyList_GetItem(names, i)); - PyObject *tup = PyList_GetItem(values, i); - if (!tup) - throw pdal_error(plang::getTraceback()); - - // Get offset. - PyObject* offset_o = PySequence_Fast_GET_ITEM(tup, 1); - if (!offset_o) - throw pdal_error(plang::getTraceback()); - offset = PyLong_AsLong(offset_o); - - // Get type. - PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0); - type = getType(dt, name); - - char byteorder = dt->byteorder; - int elsize = dt->elsize; - id = registerDim(layout, name, type); - m_fields.push_back({id, type, offset, byteorder, elsize}); - } - } -} - - -void NumpyReader::addDimensions(PointLayoutPtr layout) -{ - using namespace Dimension; - - wakeUpNumpyArray(); - createFields(layout); - - m_storeXYZ = true; - // If we already have an X dimension, we're done. - for (const Field& field : m_fields) - if (field.m_id == Id::X || field.m_id == Id::Y || field.m_id == Id::Z) - { - m_storeXYZ = false; - return; - } - - // We're storing a calculated XYZ, so register the dims. - layout->registerDim(Id::X, Type::Signed32); - if (m_ndims > 1) - { - layout->registerDim(Id::Y, Type::Signed32); - if (m_ndims > 2) - layout->registerDim(Id::Z, Type::Signed32); - } - if (m_order == Order::Row) - { - m_xIter = m_shape[m_ndims - 1]; - m_xDiv = 1; - if (m_ndims > 1) - { - m_yDiv = 1; - m_xDiv = m_xIter; - m_xIter *= m_shape[m_ndims - 2]; - m_yIter = m_shape[m_ndims - 1]; - if (m_ndims > 2) - { - m_xDiv = m_xIter; - m_yDiv = m_yIter; - m_zDiv = 1; - m_xIter *= m_shape[m_ndims - 3]; - m_yIter *= m_shape[m_ndims - 2]; - m_zIter = m_shape[m_ndims - 1]; - } - } - } - else - { - m_xIter = m_shape[0]; - m_xDiv = 1; - if (m_ndims > 1) - { - m_yIter = m_shape[0] * m_shape[1]; - m_yDiv = m_xIter; - if (m_ndims > 2) - { - m_zIter = m_shape[0] * m_shape[1] * m_shape[2]; - m_zDiv = m_yIter; - } - } - } -} - - -void NumpyReader::ready(PointTableRef table) -{ - plang::Environment::get()->set_stdout(log()->getLogStream()); - - // Set our iterators - // The location of the data pointer which the iterator may update - m_dataptr = NpyIter_GetDataPtrArray(m_iter); - - // The location of the stride which the iterator may update - m_strideptr = NpyIter_GetInnerStrideArray(m_iter); - - // The location of the inner loop size which the iterator may update - m_innersizeptr = NpyIter_GetInnerLoopSizePtr(m_iter); - - p_data = *m_dataptr; - m_chunkCount = *m_innersizeptr; - m_index = 0; - - log()->get(LogLevel::Debug) << "Initializing Numpy array for file '" << - m_filename << "'" << std::endl; - log()->get(LogLevel::Debug) << "numpy inner stride '" << - *m_strideptr << "'" << std::endl; - log()->get(LogLevel::Debug) << "numpy inner stride size '" << - *m_innersizeptr << "'" << std::endl; - log()->get(LogLevel::Debug) << "numpy number of points '" << - m_numPoints << "'" << std::endl; - log()->get(LogLevel::Debug) << "numpy number of dimensions '" << - m_ndims << "'" << std::endl; - for (npy_intp i = 0; i < m_ndims; ++i) - log()->get(LogLevel::Debug) << "numpy shape dimension number '" << - i << "' is '" << m_shape[i] <<"'" << std::endl; - - PointLayoutPtr layout = table.layout(); - MetadataNode m = layout->toMetadata(); - - pdal::Utils::toJSON(m, log()->get(LogLevel::Debug3)); - -} - -bool NumpyReader::nextPoint() -{ - // If we are at the end of this chunk, try to fetch another. Otherwise, - // just advance by the stride. - if (--m_chunkCount == 0) - { - // If we can't fetch the next ite - if (!m_iternext(m_iter)) - return false; - m_chunkCount = *m_innersizeptr; - p_data = *m_dataptr; - } - else - p_data += *m_strideptr; - return true; -} - - - - -bool NumpyReader::loadPoint(PointRef& point, point_count_t position) -{ - using namespace Dimension; - - pdal::SwitchableExtractor extractor(p_data, *m_strideptr); - - std::vector buf(*m_strideptr,0); - - float flt(0.0); - double dbl(0.0); - uint8_t u8(0); - uint16_t u16(0); - uint32_t u32(0); - uint64_t u64(0); - int8_t i8(0); - int16_t i16(0); - int32_t i32(0); - int64_t i64(0); - - for (const Field& f : m_fields) - { - if (f.m_byteorder == '>') - extractor.switchToBigEndian(); - else - extractor.switchToLittleEndian(); - - switch (f.m_type) - { - case Dimension::Type::Signed8: - extractor >> i8; - point.setField(f.m_id, i8); - break; - - case Dimension::Type::Signed16: - extractor >> i16; - point.setField(f.m_id, i16); - break; - - case Dimension::Type::Signed32: - extractor >> i32; - point.setField(f.m_id, i32); - break; - - case Dimension::Type::Signed64: - extractor >> i64; - point.setField(f.m_id, i64); - break; - - case Dimension::Type::Unsigned8: - extractor >> u8; - point.setField(f.m_id, u8); - break; - - case Dimension::Type::Unsigned16: - extractor >> u16; - point.setField(f.m_id, u16); - break; - - case Dimension::Type::Unsigned32: - extractor >> u32; - point.setField(f.m_id, u32); - break; - - case Dimension::Type::Unsigned64: - extractor >> u64; - point.setField(f.m_id, u64); - break; - - case Dimension::Type::Float: - extractor >> flt; - point.setField(f.m_id, flt); - break; - - case Dimension::Type::Double: - extractor >> dbl; - point.setField(f.m_id, dbl); - break; - - default: - // skip it - extractor.skip(f.m_elsize); - - } - - } - - if (m_storeXYZ) - { - point.setField(Dimension::Id::X, (position % m_xIter) / m_xDiv); - if (m_ndims > 1) - { - point.setField(Dimension::Id::Y, (position % m_yIter) / m_yDiv); - if (m_ndims > 2) - point.setField(Dimension::Id::Z, (position % m_zIter) / m_zDiv); - } - } - - return (nextPoint()); -} - - -bool NumpyReader::processOne(PointRef& point) -{ - if (m_index >= m_numPoints) - return false; - return loadPoint(point, m_index++); -} - - -point_count_t NumpyReader::read(PointViewPtr view, point_count_t numToRead) -{ - PointId idx = view->size(); - point_count_t numRead(0); - - while (numRead < numToRead) - { - PointRef point(*view, idx); - if (!processOne(point)) - break; - numRead++; - idx++; - } - return numRead; -} - - -void NumpyReader::done(PointTableRef) -{ - // Dereference everything we're using - - if (m_iter) - NpyIter_Deallocate(m_iter); - - Py_XDECREF(m_array); -} - - -} // namespace pdal - - diff --git a/pdal/io/NumpyReader.hpp b/pdal/io/NumpyReader.hpp deleted file mode 100644 index 2c62a1d5..00000000 --- a/pdal/io/NumpyReader.hpp +++ /dev/null @@ -1,134 +0,0 @@ -/****************************************************************************** -* Copyright (c) 2018, Howard Butler, howard@hobu.co -* -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following -* conditions are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided -* with the distribution. -* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the -* names of its contributors may be used to endorse or promote -* products derived from this software without specific prior -* written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY -* OF SUCH DAMAGE. -****************************************************************************/ - -#pragma once - -#include -#include - -#include "../plang/Invocation.hpp" - -#include -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API -#define NO_IMPORT_ARRAY -#include - -#include - -namespace pdal -{ - -class PDAL_DLL NumpyReader : public Reader, public Streamable -{ - enum class Order - { - Row, - Column - }; - friend std::ostream& operator << (std::ostream& out, - const NumpyReader::Order& order); - friend std::istream& operator >> (std::istream& in, - NumpyReader::Order& order); -public: - NumpyReader& operator=(const NumpyReader&) = delete; - NumpyReader(const NumpyReader&) = delete; - NumpyReader(); - ~NumpyReader(); - - void setArray(PyObject* array); - std::string getName() const; - -private: - virtual void initialize(); - virtual void addArgs(ProgramArgs& args); - virtual void addDimensions(PointLayoutPtr layout); - virtual void ready(PointTableRef table); - virtual point_count_t read(PointViewPtr view, point_count_t count); - virtual bool processOne(PointRef& point); - virtual void done(PointTableRef table); - - void createFields(PointLayoutPtr layout); - bool nextPoint(); - bool loadPoint(PointRef& point, point_count_t position); - void wakeUpNumpyArray(); - Dimension::Id registerDim(PointLayoutPtr layout, const std::string& name, - Dimension::Type pdalType); - void prepareFieldsArray(PointLayoutPtr layout); - void prepareRasterArray(PointLayoutPtr layout); - - // Py_XDECREF these on the way out - PyArrayObject* m_array; - NpyIter* m_iter; - NpyIter_IterNextFunc* m_iternext; - PyArray_Descr* m_dtype; - - char** m_dataptr; - char* p_data; - npy_intp m_nonzero_count; - npy_intp* m_strideptr, *m_innersizeptr; - npy_intp* m_shape; - npy_intp m_chunkCount; - point_count_t m_numPoints; - int m_numFields; - - Arg *m_orderArg; - int m_ndims; - std::string m_defaultDimension; - Order m_order; - bool m_storeXYZ; - size_t m_xIter; - size_t m_yIter; - size_t m_zIter; - size_t m_xDiv; - size_t m_yDiv; - size_t m_zDiv; - - struct Field - { - Dimension::Id m_id; - Dimension::Type m_type; - int m_offset; - char m_byteorder; - int m_elsize; - }; - std::vector m_fields; - point_count_t m_index; - - struct Args; - std::unique_ptr m_args; - -}; - -} // namespace pdal diff --git a/pdal/nlohmann/json.hpp b/pdal/nlohmann/json.hpp deleted file mode 100644 index 3c28a9d6..00000000 --- a/pdal/nlohmann/json.hpp +++ /dev/null @@ -1,20894 +0,0 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.6.1 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -SPDX-License-Identifier: MIT -Copyright (c) 2013-2019 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef INCLUDE_NLOHMANN_JSON_HPP_ -#define INCLUDE_NLOHMANN_JSON_HPP_ - -#define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 6 -#define NLOHMANN_JSON_VERSION_PATCH 1 - -#include // all_of, find, for_each -#include // assert -#include // and, not, or -#include // nullptr_t, ptrdiff_t, size_t -#include // hash, less -#include // initializer_list -#include // istream, ostream -#include // random_access_iterator_tag -#include // unique_ptr -#include // accumulate -#include // string, stoi, to_string -#include // declval, forward, move, pair, swap -#include // vector - -// #include - - -#include - -// #include - - -#include // transform -#include // array -#include // and, not -#include // forward_list -#include // inserter, front_inserter, end -#include // map -#include // string -#include // tuple, make_tuple -#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible -#include // unordered_map -#include // pair, declval -#include // valarray - -// #include - - -#include // exception -#include // runtime_error -#include // to_string - -// #include - - -#include // size_t - -namespace nlohmann -{ -namespace detail -{ -/// struct to capture the start position of the current token -struct position_t -{ - /// the total number of characters read - std::size_t chars_read_total = 0; - /// the number of characters read in the current line - std::size_t chars_read_current_line = 0; - /// the number of lines read - std::size_t lines_read = 0; - - /// conversion to size_t to preserve SAX interface - constexpr operator size_t() const - { - return chars_read_total; - } -}; - -} // namespace detail -} // namespace nlohmann - - -namespace nlohmann -{ -namespace detail -{ -//////////////// -// exceptions // -//////////////// - -/*! -@brief general exception of the @ref basic_json class - -This class is an extension of `std::exception` objects with a member @a id for -exception ids. It is used as the base class for all exceptions thrown by the -@ref basic_json class. This class can hence be used as "wildcard" to catch -exceptions. - -Subclasses: -- @ref parse_error for exceptions indicating a parse error -- @ref invalid_iterator for exceptions indicating errors with iterators -- @ref type_error for exceptions indicating executing a member function with - a wrong type -- @ref out_of_range for exceptions indicating access out of the defined range -- @ref other_error for exceptions indicating other library errors - -@internal -@note To have nothrow-copy-constructible exceptions, we internally use - `std::runtime_error` which can cope with arbitrary-length error messages. - Intermediate strings are built with static functions and then passed to - the actual constructor. -@endinternal - -@liveexample{The following code shows how arbitrary library exceptions can be -caught.,exception} - -@since version 3.0.0 -*/ -class exception : public std::exception -{ - public: - /// returns the explanatory string - const char* what() const noexcept override - { - return m.what(); - } - - /// the id of the exception - const int id; - - protected: - exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} - - static std::string name(const std::string& ename, int id_) - { - return "[json.exception." + ename + "." + std::to_string(id_) + "] "; - } - - private: - /// an exception object as storage for error messages - std::runtime_error m; -}; - -/*! -@brief exception indicating a parse error - -This exception is thrown by the library when a parse error occurs. Parse errors -can occur during the deserialization of JSON text, CBOR, MessagePack, as well -as when using JSON Patch. - -Member @a byte holds the byte index of the last read character in the input -file. - -Exceptions have ids 1xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. -json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. -json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. -json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. -json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. -json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. -json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. -json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. -json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. -json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. -json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. -json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. -json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). - -@note For an input with n bytes, 1 is the index of the first character and n+1 - is the index of the terminating null byte or the end of file. This also - holds true when reading a byte vector (CBOR or MessagePack). - -@liveexample{The following code shows how a `parse_error` exception can be -caught.,parse_error} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref out_of_range for exceptions indicating access out of the defined range -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class parse_error : public exception -{ - public: - /*! - @brief create a parse error exception - @param[in] id_ the id of the exception - @param[in] pos the position where the error occurred (or with - chars_read_total=0 if the position cannot be - determined) - @param[in] what_arg the explanatory string - @return parse_error object - */ - static parse_error create(int id_, const position_t& pos, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - position_string(pos) + ": " + what_arg; - return parse_error(id_, pos.chars_read_total, w.c_str()); - } - - static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + - ": " + what_arg; - return parse_error(id_, byte_, w.c_str()); - } - - /*! - @brief byte index of the parse error - - The byte index of the last read character in the input file. - - @note For an input with n bytes, 1 is the index of the first character and - n+1 is the index of the terminating null byte or the end of file. - This also holds true when reading a byte vector (CBOR or MessagePack). - */ - const std::size_t byte; - - private: - parse_error(int id_, std::size_t byte_, const char* what_arg) - : exception(id_, what_arg), byte(byte_) {} - - static std::string position_string(const position_t& pos) - { - return " at line " + std::to_string(pos.lines_read + 1) + - ", column " + std::to_string(pos.chars_read_current_line); - } -}; - -/*! -@brief exception indicating errors with iterators - -This exception is thrown if iterators passed to a library function do not match -the expected semantics. - -Exceptions have ids 2xx. - -name / id | example message | description ------------------------------------ | --------------- | ------------------------- -json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. -json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. -json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. -json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. -json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. -json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. -json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. -json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. -json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. -json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). - -@liveexample{The following code shows how an `invalid_iterator` exception can be -caught.,invalid_iterator} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref out_of_range for exceptions indicating access out of the defined range -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class invalid_iterator : public exception -{ - public: - static invalid_iterator create(int id_, const std::string& what_arg) - { - std::string w = exception::name("invalid_iterator", id_) + what_arg; - return invalid_iterator(id_, w.c_str()); - } - - private: - invalid_iterator(int id_, const char* what_arg) - : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating executing a member function with a wrong type - -This exception is thrown in case of a type error; that is, a library function is -executed on a JSON value whose type does not match the expected semantics. - -Exceptions have ids 3xx. - -name / id | example message | description ------------------------------ | --------------- | ------------------------- -json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. -json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. -json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t &. -json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. -json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. -json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. -json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. -json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. -json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. -json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. -json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. -json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. -json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. -json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. -json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. -json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | -json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | - -@liveexample{The following code shows how a `type_error` exception can be -caught.,type_error} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref out_of_range for exceptions indicating access out of the defined range -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class type_error : public exception -{ - public: - static type_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("type_error", id_) + what_arg; - return type_error(id_, w.c_str()); - } - - private: - type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating access out of the defined range - -This exception is thrown in case a library function is called on an input -parameter that exceeds the expected range, for instance in case of array -indices or nonexisting object keys. - -Exceptions have ids 4xx. - -name / id | example message | description -------------------------------- | --------------- | ------------------------- -json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. -json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. -json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. -json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. -json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. -json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | -json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | -json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | - -@liveexample{The following code shows how an `out_of_range` exception can be -caught.,out_of_range} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class out_of_range : public exception -{ - public: - static out_of_range create(int id_, const std::string& what_arg) - { - std::string w = exception::name("out_of_range", id_) + what_arg; - return out_of_range(id_, w.c_str()); - } - - private: - out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating other library errors - -This exception is thrown in case of errors that cannot be classified with the -other exception types. - -Exceptions have ids 5xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref out_of_range for exceptions indicating access out of the defined range - -@liveexample{The following code shows how an `other_error` exception can be -caught.,other_error} - -@since version 3.0.0 -*/ -class other_error : public exception -{ - public: - static other_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("other_error", id_) + what_arg; - return other_error(id_, w.c_str()); - } - - private: - other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // pair - -// This file contains all internal macro definitions -// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them - -// exclude unsupported compilers -#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) - #if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #endif -#endif - -// C++ language standard detection -#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 - #define JSON_HAS_CPP_17 - #define JSON_HAS_CPP_14 -#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) - #define JSON_HAS_CPP_14 -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -// disable documentation warnings on clang -#if defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdocumentation" -#endif - -// allow for portable deprecation warnings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) - #define JSON_DEPRECATED __declspec(deprecated) -#else - #define JSON_DEPRECATED -#endif - -// allow for portable nodiscard warnings -#if defined(__has_cpp_attribute) - #if __has_cpp_attribute(nodiscard) - #if defined(__clang__) && !defined(JSON_HAS_CPP_17) // issue #1535 - #define JSON_NODISCARD - #else - #define JSON_NODISCARD [[nodiscard]] - #endif - #elif __has_cpp_attribute(gnu::warn_unused_result) - #define JSON_NODISCARD [[gnu::warn_unused_result]] - #else - #define JSON_NODISCARD - #endif -#else - #define JSON_NODISCARD -#endif - -// allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) - #define JSON_THROW(exception) throw exception - #define JSON_TRY try - #define JSON_CATCH(exception) catch(exception) - #define JSON_INTERNAL_CATCH(exception) catch(exception) -#else - #include - #define JSON_THROW(exception) std::abort() - #define JSON_TRY if(true) - #define JSON_CATCH(exception) if(false) - #define JSON_INTERNAL_CATCH(exception) if(false) -#endif - -// override exception macros -#if defined(JSON_THROW_USER) - #undef JSON_THROW - #define JSON_THROW JSON_THROW_USER -#endif -#if defined(JSON_TRY_USER) - #undef JSON_TRY - #define JSON_TRY JSON_TRY_USER -#endif -#if defined(JSON_CATCH_USER) - #undef JSON_CATCH - #define JSON_CATCH JSON_CATCH_USER - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_CATCH_USER -#endif -#if defined(JSON_INTERNAL_CATCH_USER) - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER -#endif - -// manual branch prediction -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(x, 1) - #define JSON_UNLIKELY(x) __builtin_expect(x, 0) -#else - #define JSON_LIKELY(x) x - #define JSON_UNLIKELY(x) x -#endif - -/*! -@brief macro to briefly define a mapping between an enum and JSON -@def NLOHMANN_JSON_SERIALIZE_ENUM -@since version 3.4.0 -*/ -#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ - template \ - inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [e](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.first == e; \ - }); \ - j = ((it != std::end(m)) ? it : std::begin(m))->second; \ - } \ - template \ - inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [j](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.second == j; \ - }); \ - e = ((it != std::end(m)) ? it : std::begin(m))->first; \ - } - -// Ugly macros to avoid uglier copy-paste when specializing basic_json. They -// may be removed in the future once the class is split. - -#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ - template class ObjectType, \ - template class ArrayType, \ - class StringType, class BooleanType, class NumberIntegerType, \ - class NumberUnsignedType, class NumberFloatType, \ - template class AllocatorType, \ - template class JSONSerializer> - -#define NLOHMANN_BASIC_JSON_TPL \ - basic_json - -// #include - - -#include // not -#include // size_t -#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type - -namespace nlohmann -{ -namespace detail -{ -// alias templates to reduce boilerplate -template -using enable_if_t = typename std::enable_if::type; - -template -using uncvref_t = typename std::remove_cv::type>::type; - -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template -struct index_sequence -{ - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept - { - return sizeof...(Ints); - } -}; - -template -struct merge_and_renumber; - -template -struct merge_and_renumber, index_sequence> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; - -template -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; - -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; - -template -using index_sequence_for = make_index_sequence; - -// dispatch utility (taken from ranges-v3) -template struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; - -// taken from ranges-v3 -template -struct static_const -{ - static constexpr T value{}; -}; - -template -constexpr T static_const::value; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // not -#include // numeric_limits -#include // false_type, is_constructible, is_integral, is_same, true_type -#include // declval - -// #include - - -#include // random_access_iterator_tag - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template struct make_void -{ - using type = void; -}; -template using void_t = typename make_void::type; -} // namespace detail -} // namespace nlohmann - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template -struct iterator_types {}; - -template -struct iterator_types < - It, - void_t> -{ - using difference_type = typename It::difference_type; - using value_type = typename It::value_type; - using pointer = typename It::pointer; - using reference = typename It::reference; - using iterator_category = typename It::iterator_category; -}; - -// This is required as some compilers implement std::iterator_traits in a way that -// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. -template -struct iterator_traits -{ -}; - -template -struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> - : iterator_types -{ -}; - -template -struct iterator_traits::value>> -{ - using iterator_category = std::random_access_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = T*; - using reference = T&; -}; -} // namespace detail -} // namespace nlohmann - -// #include - -// #include - -// #include - - -#include - -// #include - - -// http://en.cppreference.com/w/cpp/experimental/is_detected -namespace nlohmann -{ -namespace detail -{ -struct nonesuch -{ - nonesuch() = delete; - ~nonesuch() = delete; - nonesuch(nonesuch const&) = delete; - nonesuch(nonesuch const&&) = delete; - void operator=(nonesuch const&) = delete; - void operator=(nonesuch&&) = delete; -}; - -template class Op, - class... Args> -struct detector -{ - using value_t = std::false_type; - using type = Default; -}; - -template class Op, class... Args> -struct detector>, Op, Args...> -{ - using value_t = std::true_type; - using type = Op; -}; - -template