diff --git a/.github/environment.yml b/.github/environment.yml index a917463c..348f6abd 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -2,9 +2,13 @@ name: testenv channels: - conda-forge dependencies: - - scikit-build - - numpy - - pybind11 - - pdal - - pytest - - meshio + - scikit-build-core >=1.0 + - numpy >=1.24 + - compilers + - ninja + - pybind11 >=2.12 + - libpdal-core + - pytest >=8 + - meshio >=5.3 + - geopandas + - uv diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6e797b77..7f2c7ec9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,80 +17,64 @@ defaults: jobs: build: - name: Build ${{ matrix.os }} py${{ matrix.python-version }} + name: ${{ matrix.os }} py${{ matrix.python-version }} numpy ${{ matrix.numpy-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: true matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] + numpy-version: ['2.2', '2.5'] + exclude: + - python-version: '3.14' + numpy-version: '2.2' + - python-version: '3.11' + numpy-version: '2.5' + - python-version: '3.10' + numpy-version: '2.5' steps: - name: Check out python-pdal - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Check out python-pdal-plugins - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: PDAL/python-plugins path: ./plugins + ref: main - name: Setup micromamba - uses: mamba-org/provision-with-micromamba@main + uses: conda-incubator/setup-miniconda@v3 with: + miniforge-variant: Miniforge3 + miniforge-version: latest + python-version: ${{ matrix.python-version }} + use-mamba: true + auto-update-conda: true environment-file: .github/environment.yml - extra-specs: | - python=${{ matrix.python-version }} + + - name: Install numpy ${{ matrix.numpy-version }} + run: | + mamba install -y numpy=${{ matrix.numpy-version }} - name: Install python-pdal - run: pip install -e . + run: | + uv pip install -vv . --no-deps - name: Install python-pdal-plugins working-directory: ./plugins - run: pip install -e . - - - name: Test run: | - export PDAL_DRIVER_PATH=$(python -c "import os, skbuild; print(os.path.join('plugins', skbuild.constants.SKBUILD_DIR(), 'cmake-build'))") - pdal --drivers - py.test -v test/ - - - name: Build wheel distribution - if: matrix.os != 'ubuntu-latest' - run: python setup.py --skip-cmake bdist_wheel - - - name: Build source distribution - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' - run: python setup.py sdist - - - name: Upload distribution(s) - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.os }}-py${{ matrix.python-version }} - path: ./dist/* - - publish: - runs-on: ubuntu-latest - needs: [build] - - steps: - - name: Download distributions - uses: actions/download-artifact@v2 - with: - path: ./artifacts + uv pip install -vv . --no-deps --no-build-isolation - - name: Move artifacts to dist + - name: Test run: | - mkdir dist - find ./artifacts -type f -exec mv {} ./dist \; - tree ./dist - - - name: Publish package - uses: pypa/gh-action-pypi-publish@master - if: github.event_name == 'release' && github.event.action == 'published' - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} - packages_dir: ./dist - verbose: true + export PDAL_DRIVER_PATH=$(python -m pdal --pdal-driver-path) + export PDAL_PLUGIN_PATH=$(python -m pdal --pdal-plugin-path) + echo "PDAL_DRIVER_PATH $PDAL_DRIVER_PATH" + echo "PDAL_PLUGIN_PATH $PDAL_PLUGIN_PATH" + export PDAL_DRIVER_PATH=$PDAL_PLUGIN_PATH:$PDAL_DRIVER_PATH + python -m pdal + pdal --drivers --debug + python -m pytest -v test/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..dfe8e86b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,60 @@ +name: Release + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/release.yml' + push: + branches: + - main + paths: + - '.github/workflows/release.yml' + release: + types: + - published + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/p/pdal-plugins + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@v4 + + - name: Setup environment + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-variant: Miniforge3 + miniforge-version: latest + python-version: '3.12' + use-mamba: true + auto-update-conda: true + environment-file: .github/environment.yml + + - name: Build sdist + shell: bash -l {0} + run: | + uv build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: cibw-sdist + path: dist/*.tar.gz + + - name: Publish package distributions to PyPI + if: github.event_name == 'release' && github.event.action == 'published' + uses: pypa/gh-action-pypi-publish@release/v1 + diff --git a/CHANGES.txt b/CHANGES.txt index a0d3ed2c..7e38fb77 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,53 @@ Changes -------------------------------------------------------------------------------- + +3.2.3 +................................................................................ + +Do not build and include wheels in distro + + +3.2.2 +................................................................................ + +* Implement move ctor to satisfy MSVC 2019 https://github.com/PDAL/python/commit/667f56bd0ee465f55a14636986e80b0a9cefcf14 + + +3.2.1 +................................................................................ + +* implement #129, add pandas DataFrame i/o for convenience by @hobu in + https://github.com/PDAL/python/pull/130 +* harden getMetadata and related calls from getting non-utf-8 'json' by @hobu + in https://github.com/PDAL/python/pull/140 +* ignore DataFrame test if not GeoPandas, give up on Python 3.7 builds by @hobu + in https://github.com/PDAL/python/pull/137 + +3.2.0 +................................................................................ + +* PDAL base library 2.4.0+ is required + +* CMake project name updated to pdal-python + +* `srswkt2` property added to allow fetching of SRS info + +* pip builds require cmake >= 3.11 + +* CMAKE_CXX_STANDARD set to c++17 to match PDAL 2.4.x + +* Driver and options *actually* uses the library instead of + shelling out to `pdal` application :) + +* _get_json renamed to toJSON and made public + +* Fix #119, 'json' optional kwarg put back for now + +* DEVELOPMENT_COMPONENT in CMake FindPython skipped on OSX + +* Make sure 'type' gets set when serializing to JSON + 3.1.0 ................................................................................ diff --git a/CMakeLists.txt b/CMakeLists.txt index a24d035c..05c8961d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,29 +1,40 @@ -cmake_minimum_required(VERSION 3.11.0) -project(pdal-python) +cmake_minimum_required(VERSION 3.16.0) +project(pdal-python VERSION ${SKBUILD_PROJECT_VERSION} + DESCRIPTION "PDAL Python bindings" + HOMEPAGE_URL "https://github.com/PDAL/Python") -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_BUILD_TYPE "Release") # Python-finding settings set(Python3_FIND_STRATEGY "LOCATION") set(Python3_FIND_REGISTRY "LAST") set(Python3_FIND_FRAMEWORK "LAST") -find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) -# find PDAL. Require 2.1+ -find_package(PDAL 2.4 REQUIRED) +# Development vs. Development.Module +# https://cmake.org/cmake/help/latest/module/FindPython3.html?highlight=Development.Module +if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0" AND NOT APPLE) + set(DEVELOPMENT_COMPONENT "Development.Module") +else() + set(DEVELOPMENT_COMPONENT "Development") +endif() + +# find Python3 +find_package(Python3 COMPONENTS Interpreter ${DEVELOPMENT_COMPONENT} NumPy REQUIRED) + +# find PDAL. Require 2.7+ +find_package(PDAL 2.7 REQUIRED) # find PyBind11 find_package(pybind11 REQUIRED) set(extension "libpdalpython") pybind11_add_module(${extension} MODULE - pdal/PyArray.cpp - pdal/PyPipeline.cpp - pdal/StreamableExecutor.cpp - pdal/libpdalpython.cpp + src/pdal/PyArray.cpp + src/pdal/PyPipeline.cpp + src/pdal/StreamableExecutor.cpp + src/pdal/libpdalpython.cpp ) target_include_directories(${extension} PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) target_link_libraries(${extension} PRIVATE ${PDAL_LIBRARIES}) diff --git a/README.rst b/README.rst index daa44d71..e352ca4b 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,8 @@ Additionally, you can use it to fetch `schema`_ and `metadata`_ from PDAL operat Installation -------------------------------------------------------------------------------- +**Note** The PDAL Python bindings require the PDAL base library installed. Source code can be found at https://pdal.io and `GitHub `__. + PyPI ................................................................................ @@ -18,13 +20,27 @@ PDAL Python support is installable via PyPI: pip install PDAL + +Developers can control many settings including debug builds and where the libraries are installed +using `scikit-build-core `_ settings: + +.. code-block:: + + python -m pip install \ + -Cbuild-dir=build \ + -e \ + . \ + --config-settings=cmake.build-type="Debug" \ + -vv \ + --no-deps \ + --no-build-isolation + + GitHub ................................................................................ The repository for PDAL's Python extension is available at https://github.com/PDAL/python -Python support released independently from PDAL itself as of PDAL 1.7. - Usage -------------------------------------------------------------------------------- @@ -57,6 +73,18 @@ sorts it by the ``X`` dimension: metadata = pipeline.metadata log = pipeline.log +Pass ``timing=True`` when creating a pipeline to include PDAL timing +information in ``pipeline.log`` after execution: + +.. code-block:: python + + import logging + import pdal + + pipeline = pdal.Pipeline(json, loglevel=logging.DEBUG, timing=True) + count = pipeline.execute() + timing_log = pipeline.log + Programmatic Pipeline Construction ................................................................................ @@ -74,7 +102,7 @@ Stage Objects - A stage is an instance of ``pdal.Reader``, ``pdal.Filter`` or ``pdal.Writer``. - A stage can be instantiated by passing as keyword arguments the options applicable to the respective PDAL stage. For more on PDAL stages and their - options, check the PDAL documentation on `Stage Objects `__. + options, check the PDAL documentation on `Stage Objects `__. - The ``filename`` option of ``Readers`` and ``Writers`` as well as the ``type`` option of ``Filters`` can be passed positionally as the first argument. @@ -166,7 +194,7 @@ PDAL and Python: print(len(intensity)) # 704 points # Now use pdal to clamp points that have intensity 100 <= v < 300 - pipeline = pdal.Filter.range(limits="Intensity[100:300)").pipeline(intensity) + pipeline = pdal.Filter.expression(expression="Intensity >= 100 && Intensity < 300").pipeline(intensity) print(pipeline.execute()) # 387 points clamped = pipeline.arrays[0] @@ -191,6 +219,102 @@ PDAL and Python: with tiledb.open("clamped") as a: print(a.schema) +Reading using Numpy Arrays as buffers (advanced) +................................................................................ + +It's also possible to treat the Numpy arrays passed to PDAL as buffers that are iteratively populated through +custom python functions during the execution of the pipeline. + +This may be useful in cases where you want the reading of the input data to be handled in a streamable fashion, +like for example: + +* When the total Numpy array data wouldn't fit into memory. +* To initiate execution of a streamable PDAL pipeline while the input data is still being read. + +To enable this mode, you just need to include the python populate function along with each corresponding Numpy array. + +.. code-block:: python + + # Numpy array to be used as buffer + in_buffer = np.zeros(max_chunk_size, dtype=[("X", float), ("Y", float), ("Z", float)]) + + # The function to populate the buffer iteratively + def load_next_chunk() -> int: + """ + Function called by PDAL before reading the data from the buffer. + + IMPORTANT: must return the total number of items to be read from the buffer. + The Pipeline execution will keep calling this function in a loop until 0 is returned. + """ + # + # Replace here with your code that populates the buffer and returns the number of elements to read + # + chunk_size = next_chunk.size + in_buffer[:chunk_size]["X"] = next_chunk[:]["X"] + in_buffer[:chunk_size]["Y"] = next_chunk[:]["Y"] + in_buffer[:chunk_size]["Z"] = next_chunk[:]["Z"] + + return chunk_size + + # Configure input array and handler during Pipeline initialization... + p = pdal.Pipeline(pipeline_json, arrays=[in_buffer], stream_handlers=[load_next_chunk]) + + # ...alternatively you can use the setter on an existing Pipeline + # p.inputs = [(in_buffer, load_next_chunk)] + +The following snippet provides a simple example of how to use a Numpy array as buffer to support writing through PDAL +with total control over the maximum amount of memory to use. + +Example: Streaming the read and write of a very large LAZ file with low memory footprint +......................................................................................... + + + +.. code-block:: python + + import numpy as np + import pdal + + in_chunk_size = 10_000_000 + in_pipeline = pdal.Reader.las(**{ + "filename": "in_test.laz" + }).pipeline() + + in_pipeline_it = in_pipeline.iterator(in_chunk_size).__iter__() + + out_chunk_size = 50_000_000 + out_file = "out_test.laz" + out_pipeline = pdal.Writer.las( + filename=out_file + ).pipeline() + + out_buffer = np.zeros(in_chunk_size, dtype=[("X", float), ("Y", float), ("Z", float)]) + + def load_next_chunk(): + try: + next_chunk = next(in_pipeline_it) + except StopIteration: + # Stops the streaming + return 0 + + chunk_size = next_chunk.size + out_buffer[:chunk_size]["X"] = next_chunk[:]["X"] + out_buffer[:chunk_size]["Y"] = next_chunk[:]["Y"] + out_buffer[:chunk_size]["Z"] = next_chunk[:]["Z"] + + print(f"Loaded next chunk -> {chunk_size}") + + return chunk_size + + out_pipeline.inputs = [(out_buffer, load_next_chunk)] + + out_pipeline.loglevel = 20 # INFO + count = out_pipeline.execute_streaming(out_chunk_size) + + print(f"\nWROTE - {count}") + + + Executing Streamable Pipelines ................................................................................ Streamable pipelines (pipelines that consist exclusively of streamable PDAL @@ -201,7 +325,7 @@ returns an iterator object that yields Numpy arrays of up to ``chunk_size`` size .. code-block:: python import pdal - pipeline = pdal.Reader("test/data/autzen-utm.las") | pdal.Filter.range(limits="Intensity[80:120)") + pipeline = pdal.Reader("test/data/autzen-utm.las") | pdal.Filter.expression(expression="Intensity > 80 && Intensity < 120)") for array in pipeline.iterator(chunk_size=500): print(len(array)) # or to concatenate all arrays into one @@ -293,13 +417,84 @@ USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into ti buffer.close() +Digital Terrain Model (DTM) Creation Example +................................................................................ + +The following is a script sample that can be used to create a DTM from a PDAL- +readable pointcloud. + +Method: + +#. read point cloud file +#. remove noise +#. clean up invalid values +#. classify ground points using `SMRF `__ +#. write with `GDAL writer `__ + +.. note:: If your pointcloud already has ground classified, you can skip all but + the reader and writer and achieve the same result. + +.. code-block:: python + + import pdal + + pc_path = 'https://github.com/PDAL/data/raw/refs/heads/main/autzen/autzen.laz' + out_file = 'autzen_dtm.tif' + + + # read + reader = pdal.Reader.las(pc_path) + + # remove noisy points + lownoise_filter = pdal.Filter.range( + limits='Classification![7:7]', tag='lownoise' + ) + highnoise_filter = pdal.Filter.range( + limits='Classification![18:]', tag='highnoise' + ) + + # saving incorrectly labeled returns here, some people want this, some don't + prepare_ground = pdal.Filter.assign( + value=[ + 'Classification=0', + 'ReturnNumber=1 WHERE ReturnNumber < 1', + 'NumberOfReturns=1 WHERE NumberOfReturns < 1', + ], + tag='prepare_ground_classifier', + ) + + # classify ground + smrf_classifier = pdal.Filter.smrf(tag='ground_classifier') + + # write with gdal, resolution in feet for autzen + gdal_writer = pdal.Writer.gdal( + filename=out_file, + where='Classification == 2', + data_type='float32', + resolution=10, + output_type='idw', + window_size=3, + pdal_metadata=True, + ) + + # collect pdal stages and execute pipline + pipeline = ( + reader + | lownoise_filter + | highnoise_filter + | prepare_ground + | smrf_classifier + | gdal_writer + ) + pipeline.execute() + .. _`Numpy`: http://www.numpy.org/ -.. _`schema`: http://www.pdal.io/dimensions.html -.. _`metadata`: http://www.pdal.io/development/metadata.html +.. _`schema`: http://www.pdal.org/dimensions.html +.. _`metadata`: http://www.pdal.org/development/metadata.html .. _`TileDB`: https://tiledb.com/ .. _`TileDB-PDAL integration`: https://docs.tiledb.com/geospatial/pdal -.. _`TileDB writer plugin`: https://pdal.io/stages/writers.tiledb.html +.. _`TileDB writer plugin`: https://pdal.org/stages/writers.tiledb.html .. image:: https://github.com/PDAL/python/workflows/Build/badge.svg :target: https://github.com/PDAL/python/actions?query=workflow%3ABuild @@ -307,8 +502,8 @@ USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into ti Requirements ================================================================================ -* PDAL 2.4+ -* Python >=3.7 -* Pybind11 (eg :code:`pip install pybind11[global]`) -* Numpy (eg :code:`pip install numpy`) -* scikit-build (eg :code:`pip install scikit-build`) +* PDAL 2.7+ +* Python >=3.9 +* Pybind11 >= 2.12 (eg :code:`pip install pybind11[global]`) +* Numpy >= 1.24 at runtime and >= 2.0 for isolated builds (eg :code:`pip install numpy`) +* scikit-build-core >= 1.0 (eg :code:`pip install scikit-build-core`) diff --git a/pyproject.toml b/pyproject.toml index 972b601a..52e0e6dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,97 @@ +[project] +name = "pdal" +description = "Point cloud data processing" +readme = "README.rst" +requires-python = ">=3.9" +license = {file = "LICENSE.txt"} +keywords = ["point", "cloud", "spatial"] +authors = [ + {email = "howard@hobu.co"}, + {name = "Howard Butler"} +] +maintainers = [ + {name = "Howard Butler", email = "howard@hobu.co"} +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: GIS", +] + +dependencies = [ + "numpy >= 1.24" +] + +dynamic = ["version"] + +[project.optional-dependencies] +test = [ + "geopandas", + "meshio >= 5.3", + "pandas >= 2", + "pytest >= 8", +] + +[project.urls] +homepage = "https://pdal.org" +documentation = "https://pdal.org" +repository = "https://github.com/PDAL/Python" +changelog = "https://github.com/PDAL/python/blob/main/README.rst" + [build-system] -requires = ["scikit-build", "cmake", "ninja", "numpy", "pybind11[global]"] +requires = [ + "scikit-build-core >= 1.0", + "numpy >= 2.0", + "pybind11[global] >= 2.12", +] +build-backend = "scikit_build_core.build" + + +[tool.scikit-build] +minimum-version = "build-system.requires" +build-dir = "build/{wheel_tag}" +sdist.exclude = [ + ".github", + "**/__pycache__/**", + "**/*.pyc", +] +sdist.cmake = false +sdist.inclusion-mode = "explicit" +cmake.build-type = "Release" +sdist.include = [ + "CHANGES.txt", + "CMakeLists.txt", + "LICENSE.txt", + "pyproject.toml", + "README.rst", + "src", +] +wheel.packages = ["src/pdal"] +wheel.exclude = [ + "pdal/*.cpp", + "pdal/*.hpp", + "pdal/export.hpp", + "**/__pycache__/**", + "**/*.pyc", +] +build.verbose = false +logging.level = "ERROR" + +[tool.uv] +no-build-isolation = true + +[tool.uv.pip] +no-build-isolation = true + +[[tool.dynamic-metadata]] +provider = "scikit_build_core.metadata.regex" +field = "version" +input = "src/pdal/__init__.py" diff --git a/setup.py b/setup.py deleted file mode 100644 index a3fd2a22..00000000 --- a/setup.py +++ /dev/null @@ -1,45 +0,0 @@ -from skbuild import setup - -# Get the version from the pdal module -with open("pdal/__init__.py", "r") as fp: - for line in fp: - if line.startswith("__version__"): - version = line.split("=")[1].strip().strip("\"'") - break - else: - raise ValueError("Could not determine Python package version") - -with open("README.rst", "r", encoding="utf-8") as fp: - readme = fp.read() - -with open("CHANGES.txt", "r", encoding="utf-8") as fp: - changes = fp.read() - -setup( - name="pdal", - version=version, - description="Point cloud data processing", - license="BSD", - keywords="point cloud spatial", - author="Howard Butler", - author_email="howard@hobu.co", - maintainer="Howard Butler", - maintainer_email="howard@hobu.co", - url="https://pdal.io", - long_description=readme + "\n\n" + changes, - long_description_content_type="text/x-rst", - packages=["pdal"], - install_requires=["numpy"], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Scientific/Engineering :: GIS", - ], -) diff --git a/pdal/PyArray.cpp b/src/pdal/PyArray.cpp similarity index 66% rename from pdal/PyArray.cpp rename to src/pdal/PyArray.cpp index db219f01..62b4875a 100644 --- a/pdal/PyArray.cpp +++ b/src/pdal/PyArray.cpp @@ -34,7 +34,6 @@ #include "PyArray.hpp" #include -#include namespace pdal { @@ -78,7 +77,7 @@ Dimension::Type pdalType(int t) return Type::None; } -std::string toString(PyObject *pname) +std::string pyObjectToString(PyObject *pname) { PyObject* r = PyObject_Str(pname); if (!r) @@ -90,19 +89,24 @@ std::string toString(PyObject *pname) } // unnamed namespace -Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) -{ - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); +#if NPY_ABI_VERSION < 0x02000000 + #define PyDataType_FIELDS(descr) ((descr)->fields) + #define PyDataType_NAMES(descr) ((descr)->names) +#endif +Array::Array(PyArrayObject* array, std::shared_ptr stream_handler) + : m_array(array), m_rowMajor(true), m_stream_handler(std::move(stream_handler)) +{ Py_XINCREF(array); PyArray_Descr *dtype = PyArray_DTYPE(m_array); npy_intp ndims = PyArray_NDIM(m_array); npy_intp *shape = PyArray_SHAPE(m_array); - int numFields = (dtype->fields == Py_None) ? + + PyObject* fields = PyDataType_FIELDS(dtype); + int numFields = (fields == Py_None) ? 0 : - static_cast(PyDict_Size(dtype->fields)); + static_cast(PyDict_Size(fields)); int xyz = 0; if (numFields == 0) @@ -113,7 +117,7 @@ Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) } else { - PyObject *names_dict = dtype->fields; + PyObject *names_dict = fields; PyObject *names = PyDict_Keys(names_dict); PyObject *values = PyDict_Values(names_dict); if (!names || !values) @@ -121,7 +125,7 @@ Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) for (int i = 0; i < numFields; ++i) { - std::string name = toString(PyList_GetItem(names, i)); + std::string name = python::pyObjectToString(PyList_GetItem(names, i)); if (name == "X") xyz |= 1; else if (name == "Y") @@ -160,40 +164,77 @@ Array::~Array() Py_XDECREF(m_array); } - -ArrayIter& Array::iterator() +std::shared_ptr Array::iterator() { - ArrayIter *it = new ArrayIter(m_array); - m_iterators.push_back(std::unique_ptr(it)); - return *it; + return std::make_shared(m_array, m_stream_handler); } - -ArrayIter::ArrayIter(PyArrayObject* np_array) +ArrayIter::ArrayIter(PyArrayObject* np_array, std::shared_ptr stream_handler) + : m_stream_handler(std::move(stream_handler)) { + // Create iterator m_iter = NpyIter_New(np_array, - NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, - NPY_KEEPORDER, NPY_NO_CASTING, NULL); + NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, + NPY_KEEPORDER, NPY_NO_CASTING, NULL); if (!m_iter) throw pdal_error("Unable to create numpy iterator."); + initIterator(); +} + +void ArrayIter::initIterator() +{ + // For a stream handler, first execute it to get the buffer populated and know the size of the data to iterate + int64_t stream_chunk_size = 0; + if (m_stream_handler) { + stream_chunk_size = (*m_stream_handler)(); + if (!stream_chunk_size) { + m_done = true; + return; + } + } + + // Initialize the iterator function char *itererr; m_iterNext = NpyIter_GetIterNext(m_iter, &itererr); if (!m_iterNext) { NpyIter_Deallocate(m_iter); - throw pdal_error(std::string("Unable to create numpy iterator: ") + - itererr); + m_iter = nullptr; + throw pdal_error(std::string("Unable to retrieve iteration function from numpy iterator: ") + itererr); } m_data = NpyIter_GetDataPtrArray(m_iter); - m_stride = NpyIter_GetInnerStrideArray(m_iter); - m_size = NpyIter_GetInnerLoopSizePtr(m_iter); + m_stride = *NpyIter_GetInnerStrideArray(m_iter); + m_size = *NpyIter_GetInnerLoopSizePtr(m_iter); + if (stream_chunk_size) { + // Ensure chunk size is valid and then limit iteration accordingly + if (0 < stream_chunk_size && stream_chunk_size <= m_size) { + m_size = stream_chunk_size; + } else { + throw pdal_error(std::string("Stream chunk size not in the range of array length: ") + + std::to_string(stream_chunk_size)); + } + } m_done = false; } +void ArrayIter::resetIterator() +{ + // Reset the iterator to the initial state + if (NpyIter_Reset(m_iter, NULL) != NPY_SUCCEED) { + NpyIter_Deallocate(m_iter); + m_iter = nullptr; + throw pdal_error("Unable to reset numpy iterator."); + } + + initIterator(); +} + ArrayIter::~ArrayIter() { - NpyIter_Deallocate(m_iter); + if (m_iter != nullptr) { + NpyIter_Deallocate(m_iter); + } } ArrayIter& ArrayIter::operator++() @@ -201,10 +242,15 @@ ArrayIter& ArrayIter::operator++() if (m_done) return *this; - if (--(*m_size)) - *m_data += *m_stride; - else if (!m_iterNext(m_iter)) - m_done = true; + if (--m_size) { + *m_data += m_stride; + } else if (!m_iterNext(m_iter)) { + if (m_stream_handler) { + resetIterator(); + } else { + m_done = true; + } + } return *this; } diff --git a/pdal/PyArray.hpp b/src/pdal/PyArray.hpp similarity index 73% rename from pdal/PyArray.hpp rename to src/pdal/PyArray.hpp index 91d3e535..b2aca844 100644 --- a/pdal/PyArray.hpp +++ b/src/pdal/PyArray.hpp @@ -34,48 +34,69 @@ #pragma once +#include "export.hpp" #include + +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include + #include +#include + +#include +#include namespace pdal { namespace python { + class ArrayIter; +using ArrayStreamHandler = std::function; -class PDAL_DLL Array +class PDAL_EXPORT Array { public: using Shape = std::array; using Fields = std::vector; - Array(PyArrayObject* array); + Array(PyArrayObject* array, std::shared_ptr stream_handler = {}); ~Array(); + Array(Array&& a) = default; + Array& operator=(Array&& a) = default; + + Array(const Array&) = delete; + Array() = delete; + bool rowMajor() const { return m_rowMajor; }; Shape shape() const { return m_shape; } const Fields& fields() const { return m_fields; }; - ArrayIter& iterator(); + std::shared_ptr iterator(); private: PyArrayObject* m_array; Fields m_fields; bool m_rowMajor; Shape m_shape {}; - std::vector> m_iterators; + std::shared_ptr m_stream_handler; }; -class ArrayIter +class PDAL_EXPORT ArrayIter { public: ArrayIter(const ArrayIter&) = delete; ArrayIter() = delete; - ArrayIter(PyArrayObject*); + ArrayIter(PyArrayObject*, std::shared_ptr); ~ArrayIter(); ArrayIter& operator++(); @@ -83,12 +104,16 @@ class ArrayIter char* operator*() const { return *m_data; } private: - NpyIter *m_iter; + NpyIter *m_iter = nullptr; NpyIter_IterNextFunc *m_iterNext; char **m_data; - npy_intp *m_size; - npy_intp *m_stride; + npy_intp m_size; + npy_intp m_stride; bool m_done; + + std::shared_ptr m_stream_handler; + void initIterator(); + void resetIterator(); }; } // namespace python diff --git a/pdal/PyDimension.hpp b/src/pdal/PyDimension.hpp similarity index 100% rename from pdal/PyDimension.hpp rename to src/pdal/PyDimension.hpp diff --git a/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp similarity index 90% rename from pdal/PyPipeline.cpp rename to src/pdal/PyPipeline.cpp index e219e1d4..64abe7a7 100644 --- a/pdal/PyPipeline.cpp +++ b/src/pdal/PyPipeline.cpp @@ -56,12 +56,12 @@ void CountPointTable::reset() PipelineExecutor::PipelineExecutor( - std::string const& json, std::vector> arrays, int level) + std::string const& json, std::vector> arrays, int level, bool timing) { if (level < 0 || level > 8) throw pdal_error("log level must be between 0 and 8!"); - LogPtr log(Log::makeLog("pypipeline", &m_logStream)); + LogPtr log(Log::makeLog("pypipeline", &m_logStream, timing)); log->setLevel(static_cast(level)); m_manager.setLog(log); @@ -73,17 +73,38 @@ PipelineExecutor::PipelineExecutor( } -point_count_t PipelineExecutor::execute() +point_count_t PipelineExecutor::execute(pdal::StringList allowedDims) { + if (allowedDims.size()) + { + m_manager.pointTable().layout()->setAllowedDims(allowedDims); + } + point_count_t count = m_manager.execute(); m_executed = true; return count; } +std::string PipelineExecutor::getSrsWKT2() const +{ + std::string output(""); + pdal::PointTableRef pointTable = m_manager.pointTable(); + -point_count_t PipelineExecutor::executeStream(point_count_t streamLimit) + pdal::SpatialReference srs = pointTable.spatialReference(); + output = srs.getWKT(); + + return output; +} + +point_count_t PipelineExecutor::executeStream(point_count_t streamLimit, + pdal::StringList allowedDims) { CountPointTable table(streamLimit); + if (allowedDims.size()) + { + pointTable().layout()->setAllowedDims(allowedDims); + } m_manager.executeStream(table); m_executed = true; return table.count(); @@ -164,6 +185,12 @@ MetadataNode computePreview(Stage* stage) } if (dims.size()) summary.add("dimensions", dims); + + if (!qi.m_metadata.empty() && qi.m_metadata.valid()) + { + summary.add(qi.m_metadata.clone("metadata")); + } + pdal::Utils::toJSON(summary, strm); return summary; @@ -200,16 +227,7 @@ std::string PipelineExecutor::getQuickInfo() const void PipelineExecutor::addArrayReaders(std::vector> arrays) { - // Make the symbols in pdal_base global so that they're accessible - // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, - // which means that without this, symbols in libpdal_base aren't available - // for resolution of symbols on future runtime linking. This is an issue - // on Alpine and other Linux variants that don't use UNIQUE symbols - // for C++ template statics only. Without this, you end up with multiple - // copies of template statics. -#ifndef _WIN32 - ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); -#endif + if (arrays.empty()) return; @@ -233,14 +251,20 @@ void PipelineExecutor::addArrayReaders(std::vector> array for (auto f : array->fields()) r.pushField(f); - ArrayIter& iter = array->iterator(); - auto incrementer = [&iter](PointId id) -> char * + auto arrayIter = array->iterator(); + auto incrementer = [arrayIter, firstPoint = true](PointId id) mutable -> char * { - if (! iter) + ArrayIter& iter = *arrayIter; + if (!firstPoint && iter) { + ++iter; + } else { + firstPoint = false; + } + + if (!iter) return nullptr; char *c = *iter; - ++iter; return c; }; @@ -270,6 +294,7 @@ PyObject* buildNumpyDescriptor(PointLayoutPtr layout) { return layout->dimOffset(id1) < layout->dimOffset(id2); }; + auto dims = layout->dims(); std::sort(dims.begin(), dims.end(), sortByOffset); @@ -309,8 +334,6 @@ PyObject* buildNumpyDescriptor(PointLayoutPtr layout) PyArrayObject* viewToNumpyArray(PointViewPtr view) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); PyObject* dtype_dict = buildNumpyDescriptor(view->layout()); PyArray_Descr *dtype = nullptr; @@ -333,9 +356,6 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view) PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); - // Build up a numpy dtype dictionary // // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', diff --git a/pdal/PyPipeline.hpp b/src/pdal/PyPipeline.hpp similarity index 85% rename from pdal/PyPipeline.hpp rename to src/pdal/PyPipeline.hpp index ec877400..45fa241e 100644 --- a/pdal/PyPipeline.hpp +++ b/src/pdal/PyPipeline.hpp @@ -34,7 +34,15 @@ #pragma once +#include "export.hpp" #include + +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include namespace pdal @@ -48,19 +56,21 @@ PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); class Array; -class PDAL_DLL PipelineExecutor { +class PDAL_EXPORT PipelineExecutor { public: - PipelineExecutor(std::string const& json, std::vector> arrays, int level); + PipelineExecutor(std::string const& json, std::vector> arrays, int level, bool timing); virtual ~PipelineExecutor() = default; - point_count_t execute(); - point_count_t executeStream(point_count_t streamLimit); + point_count_t execute(pdal::StringList allowedDims); + point_count_t executeStream(point_count_t streamLimit, pdal::StringList allowedDims); const PointViewSet& views() const; std::string getPipeline() const; std::string getMetadata() const; std::string getQuickInfo() const; std::string getSchema() const; + std::string getSrsWKT2() const; + PipelineManager const& getManager() const { return m_manager; } std::string getLog() const { return m_logStream.str(); } protected: diff --git a/pdal/StreamableExecutor.cpp b/src/pdal/StreamableExecutor.cpp similarity index 92% rename from pdal/StreamableExecutor.cpp rename to src/pdal/StreamableExecutor.cpp index ef9950e2..aad2d9d2 100644 --- a/pdal/StreamableExecutor.cpp +++ b/src/pdal/StreamableExecutor.cpp @@ -35,6 +35,9 @@ #include "PyPipeline.hpp" #include "StreamableExecutor.hpp" +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include #include @@ -67,8 +70,7 @@ void PythonPointTable::finalize() // create dtype auto gil = PyGILState_Ensure(); - if (_import_array() < 0) - std::cerr << "Could not import array!\n"; + PyObject *dtype_dict = buildNumpyDescriptor(&m_layout); if (PyArray_DescrConverter(dtype_dict, &m_dtype) == NPY_FAIL) throw pdal_error("Unable to create numpy dtype"); @@ -102,8 +104,8 @@ void PythonPointTable::py_resizeArray(point_count_t np) { if (src_idx != dest_idx) { - PyObject* src_item = PyArray_GETITEM(m_curArray, PyArray_GETPTR1(m_curArray, src_idx)); - PyArray_SETITEM(m_curArray, PyArray_GETPTR1(m_curArray, dest_idx), src_item); + PyObject* src_item = PyArray_GETITEM(m_curArray, (const char*) PyArray_GETPTR1(m_curArray, src_idx)); + PyArray_SETITEM(m_curArray, (char*) PyArray_GETPTR1(m_curArray, dest_idx), src_item); Py_XDECREF(src_item); } dest_idx++; @@ -184,12 +186,19 @@ char *PythonPointTable::getPoint(PointId idx) StreamableExecutor::StreamableExecutor(std::string const& json, std::vector> arrays, int level, + bool timing, point_count_t chunkSize, - int prefetch) - : PipelineExecutor(json, arrays, level) + int prefetch, + pdal::StringList allowedDims) + : PipelineExecutor(json, arrays, level, timing) , m_table(chunkSize, prefetch) , m_exc(nullptr) { + + if (allowedDims.size()) + { + m_table.layout()->setAllowedDims(allowedDims); + } m_thread.reset(new std::thread([this]() { try { diff --git a/pdal/StreamableExecutor.hpp b/src/pdal/StreamableExecutor.hpp similarity index 96% rename from pdal/StreamableExecutor.hpp rename to src/pdal/StreamableExecutor.hpp index 45d015c3..bb852755 100644 --- a/pdal/StreamableExecutor.hpp +++ b/src/pdal/StreamableExecutor.hpp @@ -80,8 +80,10 @@ class StreamableExecutor : public PipelineExecutor StreamableExecutor(std::string const& json, std::vector> arrays, int level, + bool timing, point_count_t chunkSize, - int prefetch); + int prefetch, + pdal::StringList allowedDim); ~StreamableExecutor(); MetadataNode getMetadata() { return m_table.metadata(); } diff --git a/pdal/__init__.py b/src/pdal/__init__.py similarity index 94% rename from pdal/__init__.py rename to src/pdal/__init__.py index 39e455f7..fb790e0d 100644 --- a/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ -__version__ = "3.1.2" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] +__version__ = '3.5.5' from . import libpdalpython from .drivers import inject_pdal_drivers diff --git a/src/pdal/__main__.py b/src/pdal/__main__.py new file mode 100644 index 00000000..4569e522 --- /dev/null +++ b/src/pdal/__main__.py @@ -0,0 +1,79 @@ +import sys +import os +import pathlib + +import sysconfig + +import argparse + +import pdal + +from . import __version__ + +__all__ = ["main"] + + +def __dir__() -> list[str]: + return __all__ + + +def print_driver_path(args): + if 'PDAL_DRIVER_PATH' in os.environ: + print (os.environ['PDAL_DRIVER_PATH']) + +def print_plugin_path(args): + purelib = sysconfig.get_paths()["purelib"] + + if sys.platform == "linux" or sys.platform == "linux2": + suffix = 'so' + purelib = purelib + os.path.sep + "pdal" + elif sys.platform == "darwin": + suffix = 'dylib' + purelib = purelib + os.path.sep + "pdal" + elif sys.platform == "win32": + suffix = 'dll' + purelib = purelib + os.path.sep + "bin" + + for f in pathlib.Path(purelib).glob(f'*.{suffix}'): + if 'pdal' in str(f.name): + if 'numpy' in str(f.name) or 'python' in str(f.name): + print (purelib) + return # we are done + +def print_version(args): + info = pdal.drivers.libpdalpython.getInfo() + pdal_version = info.version + plugin = info.plugin + debug = info.debug + + line = '----------------------------------------------------------------------------------------------------------------------------\n' + version = f'PDAL version {pdal_version}\nPython bindings version {__version__}\n' + driver_path = 'PDAL_DRIVER_PATH not set!' + if 'PDAL_DRIVER_PATH' in os.environ: + driver_path = os.environ['PDAL_DRIVER_PATH'] + plugin = f"Environment-set PDAL_DRIVER_PATH: {driver_path}" + output = f'{line}{version}{plugin}\n{line}\n{debug}' + print (output) + + +def main() -> None: + header = f"PDAL Python bindings {__version__} on Python {sys.version}" + + parser = argparse.ArgumentParser(description=header) + parser.add_argument('--pdal-driver-path', action='store_true', + help='print PDAL_DRIVER_PATH including Python plugin locations') + parser.add_argument('--pdal-plugin-path', action='store_true', + help='print location of PDAL Python plugins') + + args = parser.parse_args() + + if args.pdal_driver_path: + print_driver_path(args) + elif args.pdal_plugin_path: + print_plugin_path(args) + else: + print_version(args) + + +if __name__ == "__main__": + main() diff --git a/pdal/drivers.py b/src/pdal/drivers.py similarity index 74% rename from pdal/drivers.py rename to src/pdal/drivers.py index 4ca8b2ed..78d3dbb7 100644 --- a/pdal/drivers.py +++ b/src/pdal/drivers.py @@ -63,33 +63,9 @@ def factory(self) -> Callable[..., Stage]: def inject_pdal_drivers() -> None: -# drivers = libpdalpython.getDrivers() -# -# options = libpdalpython.getOptions() + drivers = libpdalpython.getDrivers() + options = libpdalpython.getOptions() - drivers = json.loads( - subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout - ) - options = dict( - json.loads( - subprocess.run( - ["pdal", "--options", "all", "--showjson"], capture_output=True - ).stdout - ) - ) - - command = 'pdal --options all --showjson' - - p = subprocess.run(shlex.split(command), - encoding='utf-8', - capture_output=True) - output, error = p.stdout, p.stderr - if p.returncode: - print (f'return code {p.returncode} error: "{error}"') - if error: - raise RuntimeError(f"Unable to run pdal --options with error '{error}'") - - options = dict( json.loads( output )) streamable = [] for d in drivers: name = d["name"] diff --git a/src/pdal/export.hpp b/src/pdal/export.hpp new file mode 100644 index 00000000..5a6c9aea --- /dev/null +++ b/src/pdal/export.hpp @@ -0,0 +1,44 @@ +/****************************************************************************** +* Copyright (c) 2025, Hobu Inc. (info@hobu.co) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + + +#include + +#ifndef PDAL_EXPORT +# define PDAL_EXPORT PDAL_DLL +#endif + +#ifndef PDAL_DLL +# define PDAL_DLL PDAL_EXPORT +#endif diff --git a/pdal/libpdalpython.cpp b/src/pdal/libpdalpython.cpp similarity index 64% rename from pdal/libpdalpython.cpp rename to src/pdal/libpdalpython.cpp index 3405f209..9ce7a939 100644 --- a/pdal/libpdalpython.cpp +++ b/src/pdal/libpdalpython.cpp @@ -1,11 +1,20 @@ #include #include #include +#include +#include #include #include #include +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + +#include + #include "PyArray.hpp" #include "PyDimension.hpp" #include "PyPipeline.hpp" @@ -29,7 +38,6 @@ namespace pdal { }; std::vector getDrivers() { - py::gil_scoped_acquire acquire; std::vector drivers; pdal::StageFactory f(false); @@ -57,10 +65,9 @@ namespace pdal { }; - std::vector getOptions() { - py::gil_scoped_acquire acquire; + py::object getOptions() { py::object json = py::module_::import("json"); - std::vector stageOptions; + py::dict stageOptions; pdal::StageFactory f; pdal::PluginManager::loadAll(); @@ -68,7 +75,6 @@ namespace pdal { for (auto name : stages) { - if ( name == "filters.info" ) continue; pdal::Stage *s = f.createStage(name); pdal::ProgramArgs args; s->addAllArgs(args); @@ -82,13 +88,12 @@ namespace pdal { try { j = json.attr("loads")(pystring); } catch (py::error_already_set &e) { - std::cout << "failed:" << name << "'" << ostr.str() << "'" <execute(); } + point_count_t execute(pdal::StringList allowedDims) { + point_count_t response(0); + { + py::gil_scoped_release release; + response = getExecutor()->execute(allowedDims); + } + return response; + } - point_count_t executeStream(point_count_t streamLimit) { - return getExecutor()->executeStream(streamLimit); + point_count_t executeStream(point_count_t streamLimit, pdal::StringList allowedDims) { + point_count_t response(0); + { + py::gil_scoped_release release; + response = getExecutor()->executeStream(streamLimit, allowedDims); + } + return response; } - std::unique_ptr iterator(int chunk_size, int prefetch) { + std::unique_ptr iterator(int chunk_size, int prefetch, pdal::StringList allowedDims) { return std::unique_ptr(new PipelineIterator( - getJson(), _inputs, _loglevel, chunk_size, prefetch + getJson(), _inputs, _loglevel, _timing, chunk_size, prefetch, allowedDims )); } - void setInputs(std::vector ndarrays) { + void setInputs(const std::vector& inputs) { _inputs.clear(); - for (const auto& ndarray: ndarrays) { - PyArrayObject* ndarray_ptr = (PyArrayObject*)ndarray.ptr(); - _inputs.push_back(std::make_shared(ndarray_ptr)); + for (const auto& input_obj: inputs) { + if (py::isinstance(input_obj)) { + // Backward compatibility for accepting list of numpy arrays + auto ndarray = input_obj.cast(); + _inputs.push_back(std::make_shared((PyArrayObject*)ndarray.ptr())); + } else { + // Now expected to be a list of pairs: (numpy array, stream handler) + auto input = input_obj.cast>(); + _inputs.push_back(std::make_shared( + (PyArrayObject*)input.first.ptr(), + input.second ? + std::make_shared(input.second) + : nullptr)); + } } delExecutor(); } @@ -175,15 +214,26 @@ namespace pdal { void setLogLevel(int level) { _loglevel = level; delExecutor(); } + bool getTiming() { return _timing; } + + void setTiming(bool timing) { _timing = timing; delExecutor(); } + std::string getLog() { return getExecutor()->getLog(); } std::string getPipeline() { return getExecutor()->getPipeline(); } + std::string getSrsWKT2() { return getExecutor()->getSrsWKT2(); } py::object getQuickInfo() { - py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); - py::str pystring(getExecutor()->getQuickInfo()); + std::string response; + { + py::gil_scoped_release release; + response = getExecutor()->getQuickInfo(); + } + py::bytes pybytes(response); + + py::str pystring ( pybytes.attr("decode")("utf-8", "ignore")); pystring.attr("strip"); py::object j; @@ -194,7 +244,16 @@ namespace pdal { } py::object getMetadata() { - return py::module_::import("json").attr("loads")(getExecutor()->getMetadata()); + py::object json = py::module_::import("json"); + + py::bytes pybytes(getExecutor()->getMetadata()); + py::str pystring ( pybytes.attr("decode")("utf-8", "ignore")); + + py::object j; + j = json.attr("loads")(pystring); + + return j; + } py::object getSchema() { @@ -220,7 +279,7 @@ namespace pdal { } std::string getJson() const { - PYBIND11_OVERRIDE_PURE_NAME(std::string, Pipeline, "_get_json", getJson); + PYBIND11_OVERRIDE_PURE_NAME(std::string, Pipeline, "toJSON", getJson); } bool hasInputs() { return !_inputs.empty(); } @@ -230,36 +289,50 @@ namespace pdal { void delExecutor() { _executor.reset(); } PipelineExecutor* getExecutor() { + // We need to acquire the GIL before we create the executor + // because this method does Python init stuff but pybind11 doesn't + // automatically encapsulate it with a gil_scoped_acquire like it + // does for all of the other methods it knows about + py::gil_scoped_acquire acquire; if (!_executor) - _executor.reset(new PipelineExecutor(getJson(), _inputs, _loglevel)); + _executor.reset(new PipelineExecutor(getJson(), _inputs, _loglevel, _timing)); return _executor.get(); } private: std::unique_ptr _executor; std::vector> _inputs; - int _loglevel; + int _loglevel = 0; + bool _timing = false; }; + + PYBIND11_MODULE(libpdalpython, m) { + _import_array(); + py::class_(m, "PipelineIterator") .def("__iter__", [](PipelineIterator &it) -> PipelineIterator& { return it; }) .def("__next__", &PipelineIterator::executeNext) .def_property_readonly("log", &PipelineIterator::getLog) .def_property_readonly("schema", &PipelineIterator::getSchema) + .def_property_readonly("srswkt2", &PipelineIterator::getSrsWKT2) .def_property_readonly("pipeline", &PipelineIterator::getPipeline) .def_property_readonly("metadata", &PipelineIterator::getMetadata); + py::class_(m, "Pipeline") .def(py::init<>()) - .def("execute", &Pipeline::execute) - .def("execute_streaming", &Pipeline::executeStream, "chunk_size"_a=10000) - .def("iterator", &Pipeline::iterator, "chunk_size"_a=10000, "prefetch"_a=0) + .def("execute", &Pipeline::execute, py::arg("allowed_dims") =py::list()) + .def("execute_streaming", &Pipeline::executeStream, "chunk_size"_a=10000, py::arg("allowed_dims") =py::list()) + .def("iterator", &Pipeline::iterator, "chunk_size"_a=10000, "prefetch"_a=0, py::arg("allowed_dims") =py::list()) .def_property("inputs", nullptr, &Pipeline::setInputs) .def_property("loglevel", &Pipeline::getLoglevel, &Pipeline::setLogLevel) + .def_property("timing", &Pipeline::getTiming, &Pipeline::setTiming) .def_property_readonly("log", &Pipeline::getLog) .def_property_readonly("schema", &Pipeline::getSchema) + .def_property_readonly("srswkt2", &Pipeline::getSrsWKT2) .def_property_readonly("pipeline", &Pipeline::getPipeline) .def_property_readonly("quickinfo", &Pipeline::getQuickInfo) .def_property_readonly("metadata", &Pipeline::getMetadata) @@ -267,14 +340,20 @@ namespace pdal { .def_property_readonly("meshes", &Pipeline::getMeshes) .def_property_readonly("_has_inputs", &Pipeline::hasInputs) .def("_copy_inputs", &Pipeline::copyInputs) - .def("_get_json", &Pipeline::getJson) + .def("toJSON", &Pipeline::getJson) .def("_del_executor", &Pipeline::delExecutor); m.def("getInfo", &getInfo); m.def("getDrivers", &getDrivers); m.def("getOptions", &getOptions); m.def("getDimensions", &getDimensions); - m.def("infer_reader_driver", &StageFactory::inferReaderDriver); - m.def("infer_writer_driver", &StageFactory::inferWriterDriver); + m.def("infer_reader_driver", &getReaderDriver); + m.def("infer_writer_driver", &getWriterDriver); + + if (pdal::Config::versionMajor() < 2) + throw pybind11::import_error("PDAL version must be >= 2.7"); + + if (pdal::Config::versionMajor() == 2 && pdal::Config::versionMinor() < 7) + throw pybind11::import_error("PDAL version must be >= 2.7"); }; }; // namespace pdal diff --git a/pdal/pipeline.py b/src/pdal/pipeline.py similarity index 61% rename from pdal/pipeline.py rename to src/pdal/pipeline.py index 118b2c50..889bc075 100644 --- a/pdal/pipeline.py +++ b/src/pdal/pipeline.py @@ -2,15 +2,26 @@ import json import logging -from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast +from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast, Callable import numpy as np +import pathlib try: from meshio import Mesh except ModuleNotFoundError: # pragma: no cover Mesh = None +try: + from pandas import DataFrame +except ModuleNotFoundError: # pragma: no cover + DataFrame = None + +try: + from geopandas import GeoDataFrame, points_from_xy +except ModuleNotFoundError: # pragma: no cover + GeoDataFrame = points_from_xy = None + from . import drivers, libpdalpython LogLevelToPDAL = { @@ -22,21 +33,51 @@ LogLevelFromPDAL = {v: k for k, v in LogLevelToPDAL.items()} +HaveFileSpecSupport = False +if libpdalpython.getInfo().major == 2 and \ + libpdalpython.getInfo().minor >= 9 or \ + libpdalpython.getInfo().major > 2: + HaveFileSpecSupport = True + + class Pipeline(libpdalpython.Pipeline): def __init__( self, spec: Union[None, str, Sequence[Stage]] = None, arrays: Sequence[np.ndarray] = (), loglevel: int = logging.ERROR, + json: Optional[str] = None, + dataframes: Sequence[DataFrame] = (), + stream_handlers: Sequence[Callable[[], int]] = (), + *, + timing: bool = False, ): + + if json: + if spec and json: + raise ValueError("provide 'spec' or 'json' arguments, not both") + spec = json + + # Convert our data frames to Numpy Structured Arrays + if dataframes: + arrays = [df.to_records() if not "geometry" in df.columns else df.drop(columns=["geometry"]).to_records() for df in dataframes] + super().__init__() self._stages: List[Stage] = [] if spec: stages = _parse_stages(spec) if isinstance(spec, str) else spec for stage in stages: self |= stage - self.inputs = arrays + + if stream_handlers: + if len(stream_handlers) != len(arrays): + raise RuntimeError("stream_handlers must match the number of specified input arrays / dataframes") + self.inputs = [(a, h) for a, h in zip(arrays, stream_handlers)] + else: + self.inputs = [(a, None) for a in arrays] + self.loglevel = loglevel + self.timing = timing def __getstate__(self): state = self.pipeline @@ -66,6 +107,14 @@ def loglevel(self, value: int) -> None: # super() property setter is not supported libpdalpython.Pipeline.loglevel.__set__(self, loglevel) + @property + def timing(self) -> bool: + return super().timing + + @timing.setter + def timing(self, value: bool) -> None: + libpdalpython.Pipeline.timing.__set__(self, bool(value)) + def __ior__(self, other: Union[Stage, Pipeline]) -> Pipeline: if isinstance(other, Stage): self._stages.append(other) @@ -86,7 +135,7 @@ def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return new def __copy__(self) -> Pipeline: - clone = self.__class__(loglevel=self.loglevel) + clone = self.__class__(loglevel=self.loglevel, timing=self.timing) clone._copy_inputs(self) clone |= self return clone @@ -106,7 +155,30 @@ def get_meshio(self, idx: int) -> Optional[Mesh]: [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], ) + def get_dataframe(self, idx: int) -> Optional[DataFrame]: + if DataFrame is None: + raise RuntimeError("Pandas support requires Pandas to be installed") + + return DataFrame(self.arrays[idx]) + + def get_geodataframe(self, idx: int, xyz: bool=False, crs: Any=None) -> Optional[GeoDataFrame]: + if GeoDataFrame is None: + raise RuntimeError("GeoPandas support requires GeoPandas to be installed") + df = DataFrame(self.arrays[idx]) + coords = [df["X"], df["Y"], df["Z"]] if xyz else [df["X"], df["Y"]] + geometry = points_from_xy(*coords) + gdf = GeoDataFrame( + df, + geometry=geometry, + crs=crs, + ) + df = coords = geometry = None + return gdf + def _get_json(self) -> str: + return self.toJSON() + + def toJSON(self) -> str: options_list = [] stage2tag: Dict[Stage, str] = {} stages = self._stages @@ -115,11 +187,16 @@ def _get_json(self) -> str: for stage in stages: stage2tag[stage] = stage.tag or _generate_tag(stage, stage2tag.values()) options = stage.options + for option in options: + if isinstance(options[option], pathlib.Path): + options[option] = str(options[option]) options["tag"] = stage2tag[stage] + options["type"] = stage.type inputs = _get_input_tags(stage, stage2tag) if inputs: options["inputs"] = inputs options_list.append(options) + return json.dumps(options_list) @@ -148,8 +225,13 @@ def inputs(self) -> List[Union[Stage, str]]: def options(self) -> Dict[str, Any]: return dict(self._options) - def pipeline(self, *arrays: np.ndarray, loglevel: int = logging.ERROR) -> Pipeline: - return Pipeline((self,), arrays, loglevel) + def pipeline( + self, + *arrays: np.ndarray, + loglevel: int = logging.ERROR, + timing: bool = False, + ) -> Pipeline: + return Pipeline((self,), arrays, loglevel=loglevel, timing=timing) def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return Pipeline((self, other)) @@ -158,7 +240,25 @@ def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: class InferableTypeStage(Stage): def __init__(self, filename: Optional[str] = None, **options: Any): if filename: - options["filename"] = filename + if isinstance(filename, dict): + if "path" not in filename: + raise ValueError(f"'path' is missing in the provided filespec: {filename}") + if HaveFileSpecSupport: + options["filename"] = filename + else: + # log that we can't pass FileSpec to PDAL + # because the library version is too old + msg = "PDAL library version is too old for FileSpec support. " \ + "Defaulting to using filename only" + logging.info(msg) + options["filename"] = filename["path"] + + else: + if HaveFileSpecSupport: + filespec = {'path':str(filename)} + options["filename"] = filespec + else: + options["filename"] = filename super().__init__(**options) @property @@ -167,7 +267,18 @@ def type(self) -> str: return super().type except KeyError: filename = self._options.get("filename") - return str(self._infer_type(filename) if filename else "") + if isinstance(filename, dict): + if "path" not in filename: + raise ValueError(f"'path' is missing in the provided filespec: {filename}") + if HaveFileSpecSupport: + path = filename.get('path') + else: + path = filename + + else: + path = str(filename) + + return str(self._infer_type(path) if filename else "") _infer_type = staticmethod(lambda filename: "") diff --git a/test/data/simple.laz b/test/data/simple.laz new file mode 100644 index 00000000..6f774c5b Binary files /dev/null and b/test/data/simple.laz differ diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 33e09324..3eac09e9 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -1,12 +1,15 @@ import json import logging import os +import re import sys +from itertools import product import numpy as np import pytest import pdal +import pathlib DATADIRECTORY = os.path.join(os.path.dirname(__file__), "data") @@ -15,6 +18,13 @@ def a_filter(ins, outs): return True +def compare_structured_arrays(arr1, arr2): + for field in arr1.dtype.names: + equal = np.all(np.equal(arr1[field], arr2[field])) + if not equal: + return False + return True + def get_pipeline(filename): with open(os.path.join(DATADIRECTORY, filename), "r") as f: if filename.endswith(".json"): @@ -27,7 +37,7 @@ def get_pipeline(filename): def test_dimensions(): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - assert 71 < len(dims) < 120 + assert len(dims) > 0 class TestPipeline: @@ -44,6 +54,21 @@ def test_construction(self, filename): assert isinstance(p, pdal.Pipeline) assert len(p.stages) == 2 + def test_timing_is_optional_public_api(self): + with open(os.path.join(DATADIRECTORY, "chip.json"), "r") as f: + pipeline_json = f.read() + + p = pdal.Pipeline(None, (), logging.ERROR, pipeline_json) + assert p.timing is False + assert p.execute() == 1065 + + with pytest.raises(TypeError): + pdal.Pipeline(None, (), logging.ERROR, pipeline_json, (), (), True) + + reader = pdal.Reader(os.path.join(DATADIRECTORY, "1.2-with-color.las")) + assert reader.pipeline().timing is False + assert reader.pipeline(timing=True).timing is True + @pytest.mark.parametrize( "pipeline", [ @@ -74,6 +99,17 @@ def test_execute_streaming(self, filename): count2 = r.execute_streaming(chunk_size=100) assert count == count2 + + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_subsetstreaming(self, filename): + """Can we fetch a subset of PDAL dimensions as a numpy array while streaming""" + r = get_pipeline(filename) + limit = ['X','Y','Z','Intensity'] + arrays = list(r.iterator(chunk_size=100,allowed_dims=limit)) + assert len(arrays) == 11 + assert len(arrays[0].dtype) == 4 + + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_execute_streaming_non_streamable(self, filename): r = get_pipeline(filename) @@ -105,6 +141,18 @@ def test_array(self, filename): assert a[0][0] == 635619.85 assert a[1064][2] == 456.92 + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_subsetarray(self, filename): + """Can we fetch a subset of PDAL dimensions as a numpy array""" + r = get_pipeline(filename) + limit = ['X','Y','Z'] + r.execute(allowed_dims=limit) + arrays = r.arrays + assert len(arrays) == 1 + assert len(arrays[0].dtype) == 3 + + + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_metadata(self, filename): """Can we fetch PDAL metadata""" @@ -132,11 +180,13 @@ def test_pipeline(self, filename): """Can we fetch PDAL pipeline string""" r = get_pipeline(filename) r.execute() - assert json.loads(r.pipeline) == { - "pipeline": [ + # filename might be an object in PDAL 2.9+ + # https://github.com/PDAL/PDAL/issues/4751 + + returned = json.loads(r.pipeline) + expected = { "pipeline": [ { - # TODO: update this after https://github.com/PDAL/PDAL/issues/3574 - "filename": f"test/data{os.sep}1.2-with-color.las", + "filename": "test/data/1.2-with-color.las", "tag": "readers_las1", "type": "readers.las", }, @@ -148,6 +198,10 @@ def test_pipeline(self, filename): }, ] } + try: + assert returned['pipeline'][0]['filename'] == "test/data/1.2-with-color.las" + except AttributeError: + assert returned['pipeline'][0]['filename']['path'] == "test/data/1.2-with-color.las" @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): @@ -245,6 +299,20 @@ def test_infer_stage_type(self): assert pdal.Writer("foo.xxx").type == "" assert pdal.Reader().type == "" assert pdal.Writer().type == "" + assert pdal.Reader({"path": "foo.las"}).type == "readers.las" + assert pdal.Writer({"path": "foo.las"}).type == "writers.las" + assert pdal.Reader({"path": "foo.xxx"}).type == "" + assert pdal.Writer({"path": "foo.xxx"}).type == "" + assert pdal.Reader({}).type == "" + assert pdal.Writer({}).type == "" + + def test_filespec(self): + """Can transit filespecs""" + spec = {'path':'junk.las', 'headers':{'header1':'header_1', 'header2':'header_2'}, 'query':{'query1':'query_1', 'query2':'query_2'}} + assert pdal.Reader(spec).type == "readers.las" + assert pdal.Reader(spec).options['filename']['path'] == "junk.las" + assert pdal.Reader(spec).options['filename']['headers']['header2'] == "header_2" + assert pdal.Writer("foo.las").type == "writers.las" def test_streamable(self): """Can we distinguish streamable from non-streamable stages and pipeline""" @@ -286,6 +354,7 @@ def test_logging(self, filename): """Can we fetch log output""" r = get_pipeline(filename) assert r.loglevel == logging.ERROR + assert r.timing is False assert r.log == "" for loglevel in logging.CRITICAL, -1: @@ -304,6 +373,17 @@ def test_logging(self, filename): assert "(pypipeline Debug) Executing pipeline in standard mode" in r.log assert "(pypipeline writers.las Debug)" in r.log + def test_logging_timing(self): + """Can we fetch log output decorated with timing information""" + with open(os.path.join(DATADIRECTORY, "chip.json"), "r") as f: + r = pdal.Pipeline(f.read(), loglevel=logging.DEBUG, timing=True) + + assert r.timing is True + count = r.execute() + assert count == 1065 + assert re.search(r"\(pypipeline readers\.las Debug [0-9.]+\)", r.log) + assert re.search(r"\(pypipeline Debug [0-9.]+\) Executing pipeline in standard mode", r.log) + @pytest.mark.skipif( not hasattr(pdal.Filter, "python"), reason="filters.python PDAL plugin is not available", @@ -338,7 +418,7 @@ def test_logging_filters_python(self, filename): reason="filters.python PDAL plugin is not available", ) def test_filters_python(self): - r = pdal.Reader("test/data/autzen-utm.las") + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) f = pdal.Filter.python(script=__file__, function="a_filter", module="anything") count = (r | f).execute() assert count == 1065 @@ -358,12 +438,29 @@ def test_only_readers(self): np.testing.assert_array_equal(np.concatenate([array1, array1]), array2) def test_quickinfo(self): - r = pdal.Reader("test/data/autzen-utm.las") + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + info = p.quickinfo + assert 'readers.las' in info.keys() + assert info['readers.las']['num_points'] == 1065 + + def test_quickinfo_offsets_scales(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"simple.laz")) p = r.pipeline() info = p.quickinfo assert 'readers.las' in info.keys() + assert 'offset_x' in info['readers.las']['metadata'].keys() + assert 'scale_x' in info['readers.las']['metadata'].keys() assert info['readers.las']['num_points'] == 1065 + def test_jsonkwarg(self): + pipeline = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")).pipeline().toJSON() + r = pdal.Pipeline(json=pipeline) + p = r.pipeline + assert 'readers.las' in p + + + class TestArrayLoad: def test_merged_arrays(self): """Can we load data from a list of arrays to PDAL""" @@ -441,6 +538,7 @@ def test_reference_counting(self): assert refcount == 1 + class TestMesh: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): @@ -472,6 +570,117 @@ def test_meshio(self, filename): assert len(triangles) == 134 assert triangles[0][0] == 29 + def test_pathlib(self): + """Can we build a pipeline using pathlib.Path as the filenames""" + path = pathlib.Path("test/data/autzen-utm.las") + read = pdal.Reader(path) + pipeline = read.pipeline() + pipeline.execute() + + +class TestDataFrame: + + @pytest.mark.skipif( + not pdal.pipeline.DataFrame, + reason="pandas is not available", + ) + def test_fetch(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + df = p.get_dataframe(0) + assert len(df) == 1065 + assert len(df.columns) == 20 + + @pytest.mark.skipif( + not pdal.pipeline.DataFrame, + reason="pandas is not available", + ) + def test_load(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + data = p.arrays[0] + df = pdal.pipeline.DataFrame + dataframes = [df(data), df(data), df(data)] + filter_intensity = """{ + "pipeline":[ + { + "type":"filters.range", + "limits":"Intensity[100:300)" + } + ] + }""" + p = pdal.Pipeline(filter_intensity, dataframes = dataframes) + p.execute() + arrays = p.arrays + assert len(arrays) == 3 + + # We copied the array three times. Sum the Intensity values + # post filtering to see if we had our intended effect + for data in arrays: + assert len(data) == 387 + assert data["Intensity"].sum() == 57684 + + +class TestGeoDataFrame: + + @pytest.mark.skipif( + not pdal.pipeline.GeoDataFrame, + reason="geopandas is not available", + ) + def test_fetch(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + record_count = p.arrays[0].shape[0] + dimension_count = len(p.arrays[0].dtype) + gdf = p.get_geodataframe(0) + gdf_xyz = p.get_geodataframe(0, xyz=True) + gdf_crs = p.get_geodataframe(0, crs="EPSG:4326") + assert len(gdf) == record_count + assert len(gdf.columns) == dimension_count + 1 + assert isinstance(gdf, pdal.pipeline.GeoDataFrame) + assert gdf.geometry.is_valid.all() + assert not gdf.geometry.is_empty.any() + assert gdf.crs is None + assert gdf.geometry.z.isna().all() + assert not gdf_xyz.geometry.z.isna().any() + assert gdf_crs.crs.srs == "EPSG:4326" + + @pytest.mark.skipif( + not pdal.pipeline.GeoDataFrame, + reason="geopandas is not available", + ) + def test_load(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + data = p.arrays[0] + gdf = pdal.pipeline.GeoDataFrame( + data, + geometry=pdal.pipeline.points_from_xy(data["X"], data["Y"], data["Z"]) + ) + dataframes = [gdf, gdf, gdf] + filter_intensity = """{ + "pipeline":[ + { + "type":"filters.range", + "limits":"Intensity[100:300)" + } + ] + }""" + p = pdal.Pipeline(filter_intensity, dataframes = dataframes) + p.execute() + arrays = p.arrays + assert len(arrays) == 3 + + # We copied the array three times. Sum the Intensity values + # post filtering to see if we had our intended effect + for data in arrays: + assert len(data) == 387 + assert data["Intensity"].sum() == 57684 + class TestPipelineIterator: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) @@ -496,7 +705,7 @@ def test_array(self, filename): arrays = list(r.iterator(chunk_size=100)) assert len(arrays) == 11 concat_array = np.concatenate(arrays) - np.testing.assert_array_equal(array, concat_array) + assert compare_structured_arrays(np.concatenate(arrays), concat_array) @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_StopIteration(self, filename): @@ -550,9 +759,7 @@ def test_merged_arrays(self): non_streaming_array = np.concatenate(p.arrays) for chunk_size in range(5, 100, 5): streaming_arrays = list(p.iterator(chunk_size=chunk_size)) - np.testing.assert_array_equal( - np.concatenate(streaming_arrays), non_streaming_array - ) + assert compare_structured_arrays(np.concatenate(streaming_arrays), non_streaming_array) @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_premature_exit(self, filename): @@ -562,9 +769,12 @@ def test_premature_exit(self, filename): assert len(r.arrays) == 1 array = r.arrays[0] + # the dtype ordering of these arrays are not going to be the + # same. Use our testing method to compare them. + for _ in range(10): for array2 in r.iterator(chunk_size=100): - np.testing.assert_array_equal(array2, array[: len(array2)]) + assert compare_structured_arrays(array2, array[: len(array2)]) break @pytest.mark.parametrize("filename", ["range.json", "range.py"]) @@ -577,3 +787,148 @@ def test_multiple_iterators(self, filename): np.testing.assert_array_equal(a1, a2) assert next(it1, None) is None assert next(it2, None) is None + + +def gen_chunk(count, random_seed = 12345): + rng = np.random.RandomState(count*random_seed) + # Generate dummy data + result = np.zeros(count, dtype=[("X", float), ("Y", float), ("Z", float)]) + result['X'][:] = rng.uniform(-2, -1, count) + result['Y'][:] = rng.uniform(1, 2, count) + result['Z'][:] = rng.uniform(3, 4, count) + return result + + +class TestPipelineInputStreams(): + + # Test cases + ONE_ARRAY_FULL = [[gen_chunk(1234)]] + MULTI_ARRAYS_FULL = [*ONE_ARRAY_FULL, [gen_chunk(4321)]] + + ONE_ARRAY_STREAMED = [[gen_chunk(10), gen_chunk(7), gen_chunk(3), gen_chunk(5), gen_chunk(1)]] + MULTI_ARRAYS_STREAMED = [*ONE_ARRAY_STREAMED, [gen_chunk(5), gen_chunk(2), gen_chunk(3), gen_chunk(1)]] + + MULTI_ARRAYS_MIXED = [ + *MULTI_ARRAYS_STREAMED, + *MULTI_ARRAYS_FULL + ] + + @pytest.mark.parametrize("in_arrays_chunks, use_setter", [ + (arrays_chunks, use_setter) for arrays_chunks, use_setter in product([ + ONE_ARRAY_FULL, MULTI_ARRAYS_FULL, + ONE_ARRAY_STREAMED, MULTI_ARRAYS_STREAMED, + MULTI_ARRAYS_MIXED + ], ['False', 'True']) + ]) + def test_pipeline_run(self, in_arrays_chunks, use_setter): + """ + Test case to validate possible usages: + - Combining "full" arrays and "streamed" ones + - Setting input arrays through the Pipeline constructor or the setter + """ + # Assuming stream mode for lists that contain more than one chunk. + # And that first chunk is the biggest of all, to simplify input buffer size creation. + in_arrays = [ + np.zeros(chunks[0].shape, chunks[0].dtype) if len(chunks) > 1 else chunks[0] + for chunks in in_arrays_chunks + ] + + def get_stream_handler(in_array, in_array_chunks): + in_array_chunks_it = iter(in_array_chunks) + def load_next_chunk(): + try: + next_chunk = next(in_array_chunks_it) + except StopIteration: + return 0 + + chunk_size = next_chunk.size + in_array[:chunk_size]["X"] = next_chunk[:]["X"] + in_array[:chunk_size]["Y"] = next_chunk[:]["Y"] + in_array[:chunk_size]["Z"] = next_chunk[:]["Z"] + + return chunk_size + + return load_next_chunk + + stream_handlers = [ + get_stream_handler(arr, chunks) if len(chunks) > 1 else None + for arr, chunks in zip(in_arrays, in_arrays_chunks) + ] + + expected_count = sum([sum([len(c) for c in chunks]) for chunks in in_arrays_chunks]) + + pipeline = """ + { + "pipeline": [{ + "type": "filters.stats" + }] + } + """ + if use_setter: + p = pdal.Pipeline(pipeline) + p.inputs = [(a, h) for a, h in zip(in_arrays, stream_handlers)] + else: + p = pdal.Pipeline(pipeline, arrays=in_arrays, stream_handlers=stream_handlers) + + count = p.execute() + out_arrays = p.arrays + assert count == expected_count + assert len(out_arrays) == len(in_arrays) + + for in_array_chunks, out_array in zip(in_arrays_chunks, out_arrays): + np.testing.assert_array_equal(out_array, np.concatenate(in_array_chunks)) + + @pytest.mark.parametrize("in_arrays, use_setter", [ + (arrays, use_setter) for arrays, use_setter in product([ + [c[0] for c in ONE_ARRAY_FULL], + [c[0] for c in MULTI_ARRAYS_FULL] + ], ['False', 'True']) + ]) + def test_pipeline_run_backward_compat(self, in_arrays, use_setter: bool): + expected_count = sum([len(a) for a in in_arrays]) + + pipeline = """ + { + "pipeline": [{ + "type": "filters.stats" + }] + } + """ + if use_setter: + p = pdal.Pipeline(pipeline) + p.inputs = in_arrays + else: + p = pdal.Pipeline(pipeline, arrays=in_arrays) + + count = p.execute() + out_arrays = p.arrays + assert count == expected_count + assert len(out_arrays) == len(in_arrays) + + for in_array, out_array in zip(in_arrays, out_arrays): + np.testing.assert_array_equal(out_array, in_array) + + @pytest.mark.parametrize("in_array, invalid_chunk_size", [ + (in_array, invalid_chunk_size) for in_array, invalid_chunk_size in product( + [gen_chunk(1234)], + [-1, 12345]) + ]) + def test_pipeline_fail_with_invalid_chunk_size(self, in_array, invalid_chunk_size): + """ + Ensure execution fails when using an invalid stream handler: + - One that returns a negative chunk size + - One that returns a chunk size bigger than the buffer capacity + """ + was_called = False + def invalid_stream_handler(): + nonlocal was_called + if was_called: + # avoid infinite loop + raise ValueError("Invalid handler should not have been called a second time") + was_called = True + return invalid_chunk_size + + p = pdal.Pipeline(arrays=[in_array], stream_handlers=[invalid_stream_handler]) + with pytest.raises(RuntimeError, + match=f"Stream chunk size not in the range of array length: {invalid_chunk_size}"): + p.execute()