From 2da8c8fba9d9bd1424e6bcd4997b6d4cbacd9e9d Mon Sep 17 00:00:00 2001 From: drewgilliam Date: Tue, 19 Apr 2022 13:22:12 -0400 Subject: [PATCH 01/59] cmake use 'Development.Module' when available (#117) --- CMakeLists.txt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a24d035c..fb2c3d85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,17 @@ set(CMAKE_BUILD_TYPE "Release") set(Python3_FIND_STRATEGY "LOCATION") set(Python3_FIND_REGISTRY "LAST") set(Python3_FIND_FRAMEWORK "LAST") -find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) + +# Development vs. Development.Module +# https://cmake.org/cmake/help/latest/module/FindPython3.html?highlight=Development.Module +if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0") + set(DEVELOPMENT_COMPONENT "Development.Module") +else() + set(DEVELOPMENT_COMPONENT "Development") +endif() + +# find Python3 +find_package(Python3 COMPONENTS Interpreter "${DEVELOPMENT_COMPONENT}" NumPy REQUIRED) # find PDAL. Require 2.1+ find_package(PDAL 2.4 REQUIRED) From 772bcf0b25abaefe85029be80d4df0b08d74da66 Mon Sep 17 00:00:00 2001 From: drewgilliam Date: Thu, 21 Apr 2022 09:43:28 -0400 Subject: [PATCH 02/59] cmake use 'Development.Module' when available (#117) From f370cb1422af9a219a41e829644db97afb312119 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 27 Sep 2022 12:59:19 -0500 Subject: [PATCH 03/59] Fix #125, make sure to put 'type' in json serialization of stages --- pdal/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 118b2c50..e389a71f 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -116,6 +116,7 @@ def _get_json(self) -> str: stage2tag[stage] = stage.tag or _generate_tag(stage, stage2tag.values()) options = stage.options options["tag"] = stage2tag[stage] + options["type"] = stage.type inputs = _get_input_tags(stage, stage2tag) if inputs: options["inputs"] = inputs From ee07fc2e7e8cd86cd3561732b48c9a55f75ddaa8 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 28 Sep 2022 12:22:49 -0500 Subject: [PATCH 04/59] 3.2.0 release prep (#127) * tweak Development.Module support to not include Apple * force cmake > 3.11 for building with pip * bump CMAKE_CXX_STANDARD to c++17 to match PDAL * fetch drivers and options directly from the library instead of shelling out to 'pdal' command * bump version no to 3.2.0 * bypass DEVELOPMENT_COMPONENT for now * compare_structured_arrays method for testing array equality now that numpy 1.23.0's deprecation has taken effect * PDAL 2.4 uses / separators all the time * Implement #123, rename _get_json to toJSON * Revert "Implement #123, rename _get_json to toJSON" This reverts commit f35c3ddbbefc34a36dd9357a866b28932859253d. * no more editable installs * Revert "Revert "Implement #123, rename _get_json to toJSON"" This reverts commit 085bce285664762f39d6e37f8c6bd9dc686ab140. * fix #119, put back json alias to Pipeline * fix #125 and set 'type' for getJSON serialization * regularize access to a pipeline's srswkt2 to fix #112 * update CHANGES * changes note * updates and changes as suggested by review --- .github/workflows/build.yml | 4 ++-- CHANGES.txt | 24 ++++++++++++++++++++++++ CMakeLists.txt | 6 +++--- pdal/PyPipeline.cpp | 11 +++++++++++ pdal/PyPipeline.hpp | 2 ++ pdal/__init__.py | 2 +- pdal/drivers.py | 28 ++-------------------------- pdal/libpdalpython.cpp | 17 +++++++++-------- pdal/pipeline.py | 10 ++++++++++ pyproject.toml | 2 +- test/test_pipeline.py | 30 +++++++++++++++++++++++------- 11 files changed, 88 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6e797b77..fb3a6c40 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,11 +44,11 @@ jobs: python=${{ matrix.python-version }} - name: Install python-pdal - run: pip install -e . + run: pip install . - name: Install python-pdal-plugins working-directory: ./plugins - run: pip install -e . + run: pip install . - name: Test run: | diff --git a/CHANGES.txt b/CHANGES.txt index a0d3ed2c..97bbbb45 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,30 @@ Changes -------------------------------------------------------------------------------- +3.2.0 +................................................................................ + +* PDAL base library 2.4.0+ is required + +* CMake project name updated to pdal-python + +* `srswkt2` property added to allow fetching of SRS info + +* pip builds require cmake >= 3.11 + +* CMAKE_CXX_STANDARD set to c++17 to match PDAL 2.4.x + +* Driver and options *actually* uses the library instead of + shelling out to `pdal` application :) + +* _get_json renamed to toJSON and made public + +* Fix #119, 'json' optional kwarg put back for now + +* DEVELOPMENT_COMPONENT in CMake FindPython skipped on OSX + +* Make sure 'type' gets set when serializing to JSON + 3.1.0 ................................................................................ diff --git a/CMakeLists.txt b/CMakeLists.txt index fb2c3d85..61046e41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.11.0) project(pdal-python) -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_BUILD_TYPE "Release") @@ -13,14 +13,14 @@ set(Python3_FIND_FRAMEWORK "LAST") # Development vs. Development.Module # https://cmake.org/cmake/help/latest/module/FindPython3.html?highlight=Development.Module -if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0") +if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18.0" AND NOT APPLE) set(DEVELOPMENT_COMPONENT "Development.Module") else() set(DEVELOPMENT_COMPONENT "Development") endif() # find Python3 -find_package(Python3 COMPONENTS Interpreter "${DEVELOPMENT_COMPONENT}" NumPy REQUIRED) +find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) # find PDAL. Require 2.1+ find_package(PDAL 2.4 REQUIRED) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index e219e1d4..35859f06 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -80,6 +80,17 @@ point_count_t PipelineExecutor::execute() return count; } +std::string PipelineExecutor::getSrsWKT2() const +{ + std::string output(""); + pdal::PointTableRef pointTable = m_manager.pointTable(); + + + pdal::SpatialReference srs = pointTable.spatialReference(); + output = srs.getWKT(); + + return output; +} point_count_t PipelineExecutor::executeStream(point_count_t streamLimit) { diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index ec877400..c4e316c6 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -61,6 +61,8 @@ class PDAL_DLL PipelineExecutor { std::string getMetadata() const; std::string getQuickInfo() const; std::string getSchema() const; + std::string getSrsWKT2() const; + PipelineManager const& getManager() const { return m_manager; } std::string getLog() const { return m_logStream.str(); } protected: diff --git a/pdal/__init__.py b/pdal/__init__.py index 39e455f7..282acba2 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.1.2" +__version__ = "3.2.0" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] from . import libpdalpython diff --git a/pdal/drivers.py b/pdal/drivers.py index 4ca8b2ed..78d3dbb7 100644 --- a/pdal/drivers.py +++ b/pdal/drivers.py @@ -63,33 +63,9 @@ def factory(self) -> Callable[..., Stage]: def inject_pdal_drivers() -> None: -# drivers = libpdalpython.getDrivers() -# -# options = libpdalpython.getOptions() + drivers = libpdalpython.getDrivers() + options = libpdalpython.getOptions() - drivers = json.loads( - subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout - ) - options = dict( - json.loads( - subprocess.run( - ["pdal", "--options", "all", "--showjson"], capture_output=True - ).stdout - ) - ) - - command = 'pdal --options all --showjson' - - p = subprocess.run(shlex.split(command), - encoding='utf-8', - capture_output=True) - output, error = p.stdout, p.stderr - if p.returncode: - print (f'return code {p.returncode} error: "{error}"') - if error: - raise RuntimeError(f"Unable to run pdal --options with error '{error}'") - - options = dict( json.loads( output )) streamable = [] for d in drivers: name = d["name"] diff --git a/pdal/libpdalpython.cpp b/pdal/libpdalpython.cpp index 3405f209..f0e0535a 100644 --- a/pdal/libpdalpython.cpp +++ b/pdal/libpdalpython.cpp @@ -57,10 +57,10 @@ namespace pdal { }; - std::vector getOptions() { + py::object getOptions() { py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); - std::vector stageOptions; + py::dict stageOptions; pdal::StageFactory f; pdal::PluginManager::loadAll(); @@ -68,7 +68,6 @@ namespace pdal { for (auto name : stages) { - if ( name == "filters.info" ) continue; pdal::Stage *s = f.createStage(name); pdal::ProgramArgs args; s->addAllArgs(args); @@ -82,13 +81,12 @@ namespace pdal { try { j = json.attr("loads")(pystring); } catch (py::error_already_set &e) { - std::cout << "failed:" << name << "'" << ostr.str() << "'" <getLog(); } std::string getPipeline() { return getExecutor()->getPipeline(); } + std::string getSrsWKT2() { return getExecutor()->getSrsWKT2(); } py::object getQuickInfo() { py::gil_scoped_acquire acquire; @@ -220,7 +219,7 @@ namespace pdal { } std::string getJson() const { - PYBIND11_OVERRIDE_PURE_NAME(std::string, Pipeline, "_get_json", getJson); + PYBIND11_OVERRIDE_PURE_NAME(std::string, Pipeline, "toJSON", getJson); } bool hasInputs() { return !_inputs.empty(); } @@ -248,6 +247,7 @@ namespace pdal { .def("__next__", &PipelineIterator::executeNext) .def_property_readonly("log", &PipelineIterator::getLog) .def_property_readonly("schema", &PipelineIterator::getSchema) + .def_property_readonly("srswkt2", &PipelineIterator::getSrsWKT2) .def_property_readonly("pipeline", &PipelineIterator::getPipeline) .def_property_readonly("metadata", &PipelineIterator::getMetadata); @@ -260,6 +260,7 @@ namespace pdal { .def_property("loglevel", &Pipeline::getLoglevel, &Pipeline::setLogLevel) .def_property_readonly("log", &Pipeline::getLog) .def_property_readonly("schema", &Pipeline::getSchema) + .def_property_readonly("srswkt2", &Pipeline::getSrsWKT2) .def_property_readonly("pipeline", &Pipeline::getPipeline) .def_property_readonly("quickinfo", &Pipeline::getQuickInfo) .def_property_readonly("metadata", &Pipeline::getMetadata) @@ -267,7 +268,7 @@ namespace pdal { .def_property_readonly("meshes", &Pipeline::getMeshes) .def_property_readonly("_has_inputs", &Pipeline::hasInputs) .def("_copy_inputs", &Pipeline::copyInputs) - .def("_get_json", &Pipeline::getJson) + .def("toJSON", &Pipeline::getJson) .def("_del_executor", &Pipeline::delExecutor); m.def("getInfo", &getInfo); m.def("getDrivers", &getDrivers); diff --git a/pdal/pipeline.py b/pdal/pipeline.py index e389a71f..f290db86 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -28,7 +28,13 @@ def __init__( spec: Union[None, str, Sequence[Stage]] = None, arrays: Sequence[np.ndarray] = (), loglevel: int = logging.ERROR, + json: Optional[str] = None ): + if json: + if spec and json: + raise ValueError("provide 'spec' or 'json' arguments, not both") + spec = json + super().__init__() self._stages: List[Stage] = [] if spec: @@ -107,6 +113,9 @@ def get_meshio(self, idx: int) -> Optional[Mesh]: ) def _get_json(self) -> str: + return self.toJSON() + + def toJSON(self) -> str: options_list = [] stage2tag: Dict[Stage, str] = {} stages = self._stages @@ -121,6 +130,7 @@ def _get_json(self) -> str: if inputs: options["inputs"] = inputs options_list.append(options) + return json.dumps(options_list) diff --git a/pyproject.toml b/pyproject.toml index 972b601a..db01ea67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,2 @@ [build-system] -requires = ["scikit-build", "cmake", "ninja", "numpy", "pybind11[global]"] +requires = ["scikit-build", "cmake>=3.11", "ninja", "numpy", "pybind11[global]"] diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 33e09324..6675c428 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -15,6 +15,13 @@ def a_filter(ins, outs): return True +def compare_structured_arrays(arr1, arr2): + for field in arr1.dtype.names: + equal = np.all(np.equal(arr1[field], arr2[field])) + if not equal: + return False + return True + def get_pipeline(filename): with open(os.path.join(DATADIRECTORY, filename), "r") as f: if filename.endswith(".json"): @@ -135,8 +142,7 @@ def test_pipeline(self, filename): assert json.loads(r.pipeline) == { "pipeline": [ { - # TODO: update this after https://github.com/PDAL/PDAL/issues/3574 - "filename": f"test/data{os.sep}1.2-with-color.las", + "filename": "test/data/1.2-with-color.las", "tag": "readers_las1", "type": "readers.las", }, @@ -364,6 +370,14 @@ def test_quickinfo(self): assert 'readers.las' in info.keys() assert info['readers.las']['num_points'] == 1065 + def test_jsonkwarg(self): + pipeline = pdal.Reader("test/data/autzen-utm.las").pipeline().toJSON() + r = pdal.Pipeline(json=pipeline) + p = r.pipeline + assert 'readers.las' in p + + + class TestArrayLoad: def test_merged_arrays(self): """Can we load data from a list of arrays to PDAL""" @@ -441,6 +455,7 @@ def test_reference_counting(self): assert refcount == 1 + class TestMesh: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): @@ -496,7 +511,7 @@ def test_array(self, filename): arrays = list(r.iterator(chunk_size=100)) assert len(arrays) == 11 concat_array = np.concatenate(arrays) - np.testing.assert_array_equal(array, concat_array) + assert compare_structured_arrays(np.concatenate(arrays), concat_array) @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_StopIteration(self, filename): @@ -550,9 +565,7 @@ def test_merged_arrays(self): non_streaming_array = np.concatenate(p.arrays) for chunk_size in range(5, 100, 5): streaming_arrays = list(p.iterator(chunk_size=chunk_size)) - np.testing.assert_array_equal( - np.concatenate(streaming_arrays), non_streaming_array - ) + assert compare_structured_arrays(np.concatenate(streaming_arrays), non_streaming_array) @pytest.mark.parametrize("filename", ["range.json", "range.py"]) def test_premature_exit(self, filename): @@ -562,9 +575,12 @@ def test_premature_exit(self, filename): assert len(r.arrays) == 1 array = r.arrays[0] + # the dtype ordering of these arrays are not going to be the + # same. Use our testing method to compare them. + for _ in range(10): for array2 in r.iterator(chunk_size=100): - np.testing.assert_array_equal(array2, array[: len(array2)]) + assert compare_structured_arrays(array2, array[: len(array2)]) break @pytest.mark.parametrize("filename", ["range.json", "range.py"]) From 88f548ec9ee90f701860ccc0cc49dbe3aa42e83d Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 13 Oct 2022 13:50:56 -0500 Subject: [PATCH 05/59] implement #129, add pandas DataFrame i/o for convenience (#130) --- .github/environment.yml | 1 + pdal/pipeline.py | 20 +++++++++++++++++- test/test_pipeline.py | 45 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index a917463c..27ae613c 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -8,3 +8,4 @@ dependencies: - pdal - pytest - meshio + - pandas diff --git a/pdal/pipeline.py b/pdal/pipeline.py index f290db86..377a517b 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -11,6 +11,11 @@ except ModuleNotFoundError: # pragma: no cover Mesh = None +try: + from pandas import DataFrame +except ModuleNotFoundError: # pragma: no cover + DataFrame = None + from . import drivers, libpdalpython LogLevelToPDAL = { @@ -28,13 +33,19 @@ def __init__( spec: Union[None, str, Sequence[Stage]] = None, arrays: Sequence[np.ndarray] = (), loglevel: int = logging.ERROR, - json: Optional[str] = None + json: Optional[str] = None, + dataframes: Sequence[DataFrame] = (), ): + if json: if spec and json: raise ValueError("provide 'spec' or 'json' arguments, not both") spec = json + # Convert our data frames to Numpy Structured Arrays + if dataframes: + arrays = [df.to_records() for df in dataframes] + super().__init__() self._stages: List[Stage] = [] if spec: @@ -112,6 +123,13 @@ def get_meshio(self, idx: int) -> Optional[Mesh]: [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], ) + + def get_dataframe(self, idx: int) -> Optional[DataFrame]: + if DataFrame is None: + raise RuntimeError("Pandas support requires Pandas to be installed") + + return DataFrame(self.arrays[idx]) + def _get_json(self) -> str: return self.toJSON() diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 6675c428..614991d0 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -344,7 +344,7 @@ def test_logging_filters_python(self, filename): reason="filters.python PDAL plugin is not available", ) def test_filters_python(self): - r = pdal.Reader("test/data/autzen-utm.las") + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) f = pdal.Filter.python(script=__file__, function="a_filter", module="anything") count = (r | f).execute() assert count == 1065 @@ -364,14 +364,14 @@ def test_only_readers(self): np.testing.assert_array_equal(np.concatenate([array1, array1]), array2) def test_quickinfo(self): - r = pdal.Reader("test/data/autzen-utm.las") + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) p = r.pipeline() info = p.quickinfo assert 'readers.las' in info.keys() assert info['readers.las']['num_points'] == 1065 def test_jsonkwarg(self): - pipeline = pdal.Reader("test/data/autzen-utm.las").pipeline().toJSON() + pipeline = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")).pipeline().toJSON() r = pdal.Pipeline(json=pipeline) p = r.pipeline assert 'readers.las' in p @@ -487,6 +487,45 @@ def test_meshio(self, filename): assert len(triangles) == 134 assert triangles[0][0] == 29 +class TestDataFrame: + + @pytest.mark.skipif( + not pdal.pipeline.DataFrame, + reason="pandas is not available", + ) + def test_fetch(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + df = p.get_dataframe(0) + assert df.size == 17040 + + def test_load(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + data = p.arrays[0] + df = pdal.pipeline.DataFrame + dataframes = [df(data), df(data), df(data)] + filter_intensity = """{ + "pipeline":[ + { + "type":"filters.range", + "limits":"Intensity[100:300)" + } + ] + }""" + p = pdal.Pipeline(filter_intensity, dataframes = dataframes) + p.execute() + arrays = p.arrays + assert len(arrays) == 3 + + # We copied the array three times. Sum the Intensity values + # post filtering to see if we had our intended effect + for data in arrays: + assert len(data) == 387 + assert data["Intensity"].sum() == 57684 + class TestPipelineIterator: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) From fd1e922b5177321a8e95a0de77f86c61dc2c2592 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sun, 27 Nov 2022 20:11:33 -0600 Subject: [PATCH 06/59] harden getMetadata and related calls from getting non-utf-8 'json' (#140) * harden getMetadata and related calls from getting non-utf-8 'json' from PDAL * whoops * lern2type * py.test no longer always available on conda-forge packages * editable install * arg order * no editable * explicit pytest install * ditch 3.7 and 3.8 * pip install --upgrade --force-reinstall pytest * shut off embed for now * test fixes * turn on embed tests * update pypa/gh-action-pypi-publish@release/v1 --- .github/workflows/build.yml | 7 +++++-- pdal/__init__.py | 2 +- pdal/libpdalpython.cpp | 20 ++++++++++++++++---- test/test_pipeline.py | 6 +++++- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb3a6c40..371b3fe4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: fail-fast: true matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11'] steps: - name: Check out python-pdal @@ -50,6 +50,9 @@ jobs: working-directory: ./plugins run: pip install . + - name: Install pytest + run: pip install --upgrade --force-reinstall pytest + - name: Test run: | export PDAL_DRIVER_PATH=$(python -c "import os, skbuild; print(os.path.join('plugins', skbuild.constants.SKBUILD_DIR(), 'cmake-build'))") @@ -87,7 +90,7 @@ jobs: tree ./dist - name: Publish package - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 if: github.event_name == 'release' && github.event.action == 'published' with: user: __token__ diff --git a/pdal/__init__.py b/pdal/__init__.py index 282acba2..902e1035 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.2.0" +__version__ = "3.2.1" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] from . import libpdalpython diff --git a/pdal/libpdalpython.cpp b/pdal/libpdalpython.cpp index f0e0535a..7543249c 100644 --- a/pdal/libpdalpython.cpp +++ b/pdal/libpdalpython.cpp @@ -134,8 +134,9 @@ namespace pdal { MetadataNode root = (StreamableExecutor::getMetadata()).clone("metadata"); pdal::Utils::toJSON(root, strm); - py::str pystring(strm.str()); - pystring.attr("strip"); + + py::bytes pybytes(strm.str()); + py::str pystring ( pybytes.attr("decode")("utf-8", "ignore")); py::object j; j = json.attr("loads")(pystring); @@ -182,7 +183,9 @@ namespace pdal { py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); - py::str pystring(getExecutor()->getQuickInfo()); + py::bytes pybytes(getExecutor()->getQuickInfo()); + + py::str pystring ( pybytes.attr("decode")("utf-8", "ignore")); pystring.attr("strip"); py::object j; @@ -193,7 +196,16 @@ namespace pdal { } py::object getMetadata() { - return py::module_::import("json").attr("loads")(getExecutor()->getMetadata()); + py::object json = py::module_::import("json"); + + py::bytes pybytes(getExecutor()->getMetadata()); + py::str pystring ( pybytes.attr("decode")("utf-8", "ignore")); + + py::object j; + j = json.attr("loads")(pystring); + + return j; + } py::object getSchema() { diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 614991d0..a07abc06 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -34,7 +34,7 @@ def get_pipeline(filename): def test_dimensions(): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - assert 71 < len(dims) < 120 + assert 71 < len(dims) < 122 class TestPipeline: @@ -500,6 +500,10 @@ def test_fetch(self): df = p.get_dataframe(0) assert df.size == 17040 + @pytest.mark.skipif( + not pdal.pipeline.DataFrame, + reason="pandas is not available", + ) def test_load(self): r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) p = r.pipeline() From 64b5307688b718006dcacc0e9eec1e9bfdfee86f Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 5 Dec 2022 14:39:42 -0600 Subject: [PATCH 07/59] ignore DataFrame test if not GeoPandas, give up on Python 3.7 builds (#137) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 371b3fe4..a6807e98 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.9', '3.10', '3.11'] - + steps: - name: Check out python-pdal uses: actions/checkout@v2 From d604647f208a294708e381626d23967eeb112895 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 5 Dec 2022 14:43:36 -0600 Subject: [PATCH 08/59] list changes for 3.2.1 --- CHANGES.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 97bbbb45..b8cd1945 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,17 @@ Changes -------------------------------------------------------------------------------- + +3.2.1 +................................................................................ + +* implement #129, add pandas DataFrame i/o for convenience by @hobu in + https://github.com/PDAL/python/pull/130 +* harden getMetadata and related calls from getting non-utf-8 'json' by @hobu + in https://github.com/PDAL/python/pull/140 +* ignore DataFrame test if not GeoPandas, give up on Python 3.7 builds by @hobu + in https://github.com/PDAL/python/pull/137 + 3.2.0 ................................................................................ From 667f56bd0ee465f55a14636986e80b0a9cefcf14 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 16 Jan 2023 14:24:00 -0600 Subject: [PATCH 09/59] add move ctor for pdal::python::Array to satisfy newer MSVC --- pdal/PyArray.cpp | 2 +- pdal/PyArray.hpp | 9 +++++++++ pdal/__init__.py | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pdal/PyArray.cpp b/pdal/PyArray.cpp index db219f01..04c424bd 100644 --- a/pdal/PyArray.cpp +++ b/pdal/PyArray.cpp @@ -164,7 +164,7 @@ Array::~Array() ArrayIter& Array::iterator() { ArrayIter *it = new ArrayIter(m_array); - m_iterators.push_back(std::unique_ptr(it)); + m_iterators.emplace_back((it)); return *it; } diff --git a/pdal/PyArray.hpp b/pdal/PyArray.hpp index 91d3e535..f0401f00 100644 --- a/pdal/PyArray.hpp +++ b/pdal/PyArray.hpp @@ -38,6 +38,9 @@ #include #include +#include +#include + namespace pdal { namespace python @@ -55,6 +58,12 @@ class PDAL_DLL Array Array(PyArrayObject* array); ~Array(); + Array(Array&& a) = default; + Array& operator=(Array&& a) = default; + + Array(const Array&) = delete; + Array() = delete; + bool rowMajor() const { return m_rowMajor; }; Shape shape() const { return m_shape; } const Fields& fields() const { return m_fields; }; diff --git a/pdal/__init__.py b/pdal/__init__.py index 902e1035..124c9e07 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.2.1" +__version__ = "3.2.2" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] from . import libpdalpython From 3bfc5b39205205fee9998103872e0a44ddf95337 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 16 Jan 2023 14:33:49 -0600 Subject: [PATCH 10/59] list changes --- CHANGES.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index b8cd1945..0cb7bf40 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,12 @@ Changes -------------------------------------------------------------------------------- +3.2.2 +................................................................................ + +* Implement move ctor to satisfy MSVC 2019 https://github.com/PDAL/python/commit/667f56bd0ee465f55a14636986e80b0a9cefcf14 + + 3.2.1 ................................................................................ From 3339b3193029e6ad856e79e67befa01a3c1eb6f5 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 9 Mar 2023 22:13:38 -0600 Subject: [PATCH 11/59] bump to 3.2.3, do not include wheels in builds --- .github/workflows/build.yml | 6 +----- CHANGES.txt | 6 ++++++ CMakeLists.txt | 2 +- pdal/__init__.py | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6807e98..47957c77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.9', '3.10', '3.11'] - + steps: - name: Check out python-pdal uses: actions/checkout@v2 @@ -59,10 +59,6 @@ jobs: pdal --drivers py.test -v test/ - - name: Build wheel distribution - if: matrix.os != 'ubuntu-latest' - run: python setup.py --skip-cmake bdist_wheel - - name: Build source distribution if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' run: python setup.py sdist diff --git a/CHANGES.txt b/CHANGES.txt index 0cb7bf40..7e38fb77 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,6 +2,12 @@ Changes -------------------------------------------------------------------------------- +3.2.3 +................................................................................ + +Do not build and include wheels in distro + + 3.2.2 ................................................................................ diff --git a/CMakeLists.txt b/CMakeLists.txt index 61046e41..402033d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ else() endif() # find Python3 -find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED) +find_package(Python3 COMPONENTS Interpreter ${DEVELOPMENT_COMPONENT} NumPy REQUIRED) # find PDAL. Require 2.1+ find_package(PDAL 2.4 REQUIRED) diff --git a/pdal/__init__.py b/pdal/__init__.py index 124c9e07..d032690a 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.2.2" +__version__ = "3.2.3" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] from . import libpdalpython From 1cb6e1ca133d6c01f6bed2cddcbb7df3b16038ba Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 10 Mar 2023 07:55:13 -0600 Subject: [PATCH 12/59] libcxx on osx --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47957c77..16b62236 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,6 +39,10 @@ jobs: - name: Setup micromamba uses: mamba-org/provision-with-micromamba@main with: + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + auto-update-conda: true environment-file: .github/environment.yml extra-specs: | python=${{ matrix.python-version }} @@ -46,6 +50,10 @@ jobs: - name: Install python-pdal run: pip install . + - name: libcxx=14 for OSX + if: matrix.os == 'macos-latest' + run: mamba install -c conda-forge libcxx=14 -y + - name: Install python-pdal-plugins working-directory: ./plugins run: pip install . From 2c07422f9c43875a6cf25e69efaa3b3812a11787 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 10 Mar 2023 09:54:11 -0600 Subject: [PATCH 13/59] mamba'ing --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16b62236..53d508bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,7 @@ jobs: path: ./plugins - name: Setup micromamba - uses: mamba-org/provision-with-micromamba@main + uses: conda-incubator/setup-miniconda@v2 with: miniforge-variant: Mambaforge miniforge-version: latest From 92a5f88198b2aa2882ed2cad9ef44dfbc5f15b4d Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 10 Mar 2023 12:20:31 -0600 Subject: [PATCH 14/59] remove libcxx pin, needs to be in pdal proper --- .github/workflows/build.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53d508bc..8a4c4d5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,10 +50,6 @@ jobs: - name: Install python-pdal run: pip install . - - name: libcxx=14 for OSX - if: matrix.os == 'macos-latest' - run: mamba install -c conda-forge libcxx=14 -y - - name: Install python-pdal-plugins working-directory: ./plugins run: pip install . From 7c0a457c98733d86120b14ccc12872f22a2733f3 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sat, 11 Mar 2023 07:35:42 -0600 Subject: [PATCH 15/59] trigger ci From 73b46903841c2dcf3d4af0727214fb9066e8a342 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sat, 11 Mar 2023 07:56:07 -0600 Subject: [PATCH 16/59] drop pytest reinstall --- .github/workflows/build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a4c4d5c..dcfe8734 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,9 +54,6 @@ jobs: working-directory: ./plugins run: pip install . - - name: Install pytest - run: pip install --upgrade --force-reinstall pytest - - name: Test run: | export PDAL_DRIVER_PATH=$(python -c "import os, skbuild; print(os.path.join('plugins', skbuild.constants.SKBUILD_DIR(), 'cmake-build'))") From dfdc448b6a20789045c5d5114f9c68bb72967176 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 7 Aug 2023 12:18:22 -0500 Subject: [PATCH 17/59] support pathlib.Path as option types for 'filename' #148 (#149) --- pdal/libpdalpython.cpp | 15 +++++++++++++-- pdal/pipeline.py | 4 ++++ test/test_pipeline.py | 9 +++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pdal/libpdalpython.cpp b/pdal/libpdalpython.cpp index 7543249c..72b1d3a1 100644 --- a/pdal/libpdalpython.cpp +++ b/pdal/libpdalpython.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -108,6 +109,16 @@ namespace pdal { return dims; }; + std::string getReaderDriver(std::filesystem::path const& p) + { + return StageFactory::inferReaderDriver(p.string()); + } + + std::string getWriterDriver(std::filesystem::path const& p) + { + return StageFactory::inferWriterDriver(p.string()); + } + using pdal::python::PipelineExecutor; using pdal::python::StreamableExecutor; @@ -286,8 +297,8 @@ namespace pdal { m.def("getDrivers", &getDrivers); m.def("getOptions", &getOptions); m.def("getDimensions", &getDimensions); - m.def("infer_reader_driver", &StageFactory::inferReaderDriver); - m.def("infer_writer_driver", &StageFactory::inferWriterDriver); + m.def("infer_reader_driver", &getReaderDriver); + m.def("infer_writer_driver", &getWriterDriver); }; }; // namespace pdal diff --git a/pdal/pipeline.py b/pdal/pipeline.py index 377a517b..c13a6d2c 100644 --- a/pdal/pipeline.py +++ b/pdal/pipeline.py @@ -5,6 +5,7 @@ from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast import numpy as np +import pathlib try: from meshio import Mesh @@ -142,6 +143,9 @@ def toJSON(self) -> str: for stage in stages: stage2tag[stage] = stage.tag or _generate_tag(stage, stage2tag.values()) options = stage.options + for option in options: + if isinstance(options[option], pathlib.Path): + options[option] = str(options[option]) options["tag"] = stage2tag[stage] options["type"] = stage.type inputs = _get_input_tags(stage, stage2tag) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index a07abc06..2459f3d2 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -7,6 +7,7 @@ import pytest import pdal +import pathlib DATADIRECTORY = os.path.join(os.path.dirname(__file__), "data") @@ -487,6 +488,14 @@ def test_meshio(self, filename): assert len(triangles) == 134 assert triangles[0][0] == 29 + def test_pathlib(self): + """Can we build a pipeline using pathlib.Path as the filenames""" + path = pathlib.Path("test/data/autzen-utm.las") + read = pdal.Reader(path) + pipeline = read.pipeline() + pipeline.execute() + + class TestDataFrame: @pytest.mark.skipif( From cd51a2252255b27265d2c8a0fd10759c18316c8a Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 3 Jan 2024 13:39:55 -0600 Subject: [PATCH 18/59] release the Python GIL for long(er) running processes (#156) * release the Python GIL for long(er) running processes like execute and preview * update ci * print driver load debug * use conda forge compilers * ci tweaking * use main branch of python-plugins now --- .github/environment.yml | 1 + .github/workflows/build.yml | 18 ++++++++---------- pdal/libpdalpython.cpp | 30 +++++++++++++++++++++++++++--- test/test_pipeline.py | 5 +++-- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index 27ae613c..c12722d8 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -4,6 +4,7 @@ channels: dependencies: - scikit-build - numpy + - compilers - pybind11 - pdal - pytest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dcfe8734..bc5c8969 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,28 +24,26 @@ jobs: fail-fast: true matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Check out python-pdal - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Check out python-pdal-plugins - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: PDAL/python-plugins path: ./plugins - name: Setup micromamba - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest use-mamba: true auto-update-conda: true environment-file: .github/environment.yml - extra-specs: | - python=${{ matrix.python-version }} - name: Install python-pdal run: pip install . @@ -57,15 +55,15 @@ jobs: - name: Test run: | export PDAL_DRIVER_PATH=$(python -c "import os, skbuild; print(os.path.join('plugins', skbuild.constants.SKBUILD_DIR(), 'cmake-build'))") - pdal --drivers + pdal --drivers --debug py.test -v test/ - name: Build source distribution - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' run: python setup.py sdist - name: Upload distribution(s) - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ matrix.os }}-py${{ matrix.python-version }} path: ./dist/* @@ -76,7 +74,7 @@ jobs: steps: - name: Download distributions - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: path: ./artifacts diff --git a/pdal/libpdalpython.cpp b/pdal/libpdalpython.cpp index 72b1d3a1..a7b5147a 100644 --- a/pdal/libpdalpython.cpp +++ b/pdal/libpdalpython.cpp @@ -94,6 +94,7 @@ namespace pdal { }; std::vector getDimensions() { + py::gil_scoped_acquire acquire; py::object np = py::module_::import("numpy"); py::object dtype = np.attr("dtype"); std::vector dims; @@ -111,11 +112,13 @@ namespace pdal { std::string getReaderDriver(std::filesystem::path const& p) { + py::gil_scoped_acquire acquire; return StageFactory::inferReaderDriver(p.string()); } std::string getWriterDriver(std::filesystem::path const& p) { + py::gil_scoped_acquire acquire; return StageFactory::inferWriterDriver(p.string()); } @@ -139,6 +142,7 @@ namespace pdal { } py::object getMetadata() { + py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); std::stringstream strm; @@ -160,10 +164,23 @@ namespace pdal { class Pipeline { public: - point_count_t execute() { return getExecutor()->execute(); } + point_count_t execute() { + point_count_t response(0); + { + py::gil_scoped_release release; + response = getExecutor()->execute(); + } + return response; + + } point_count_t executeStream(point_count_t streamLimit) { - return getExecutor()->executeStream(streamLimit); + point_count_t response(0); + { + py::gil_scoped_release release; + response = getExecutor()->executeStream(streamLimit); + } + return response; } std::unique_ptr iterator(int chunk_size, int prefetch) { @@ -173,6 +190,7 @@ namespace pdal { } void setInputs(std::vector ndarrays) { + py::gil_scoped_acquire acquire; _inputs.clear(); for (const auto& ndarray: ndarrays) { PyArrayObject* ndarray_ptr = (PyArrayObject*)ndarray.ptr(); @@ -194,7 +212,12 @@ namespace pdal { py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); - py::bytes pybytes(getExecutor()->getQuickInfo()); + std::string response; + { + py::gil_scoped_release release; + response = getExecutor()->getQuickInfo(); + } + py::bytes pybytes(response); py::str pystring ( pybytes.attr("decode")("utf-8", "ignore")); pystring.attr("strip"); @@ -252,6 +275,7 @@ namespace pdal { void delExecutor() { _executor.reset(); } PipelineExecutor* getExecutor() { + py::gil_scoped_acquire acquire; if (!_executor) _executor.reset(new PipelineExecutor(getJson(), _inputs, _loglevel)); return _executor.get(); diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 2459f3d2..5a40b58e 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -35,7 +35,7 @@ def get_pipeline(filename): def test_dimensions(): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - assert 71 < len(dims) < 122 + assert len(dims) > 0 class TestPipeline: @@ -507,7 +507,8 @@ def test_fetch(self): p = r.pipeline() p.execute() df = p.get_dataframe(0) - assert df.size == 17040 + assert len(df) == 1065 + assert len(df.columns) == 20 @pytest.mark.skipif( not pdal.pipeline.DataFrame, From dd75a383624989b4e47e3e874ee7a57062ab95d7 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 3 Jan 2024 13:43:28 -0600 Subject: [PATCH 19/59] bump version to 3.3.0 for release --- CMakeLists.txt | 2 +- README.rst | 4 ++-- pdal/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 402033d8..b45f0c35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ endif() find_package(Python3 COMPONENTS Interpreter ${DEVELOPMENT_COMPONENT} NumPy REQUIRED) # find PDAL. Require 2.1+ -find_package(PDAL 2.4 REQUIRED) +find_package(PDAL 2.5 REQUIRED) # find PyBind11 find_package(pybind11 REQUIRED) diff --git a/README.rst b/README.rst index daa44d71..1a2d9de6 100644 --- a/README.rst +++ b/README.rst @@ -307,8 +307,8 @@ USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into ti Requirements ================================================================================ -* PDAL 2.4+ -* Python >=3.7 +* PDAL 2.5+ +* Python >=3.9 * Pybind11 (eg :code:`pip install pybind11[global]`) * Numpy (eg :code:`pip install numpy`) * scikit-build (eg :code:`pip install scikit-build`) diff --git a/pdal/__init__.py b/pdal/__init__.py index d032690a..1f05ff3d 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__ = "3.2.3" +__version__ = "3.3.0" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] from . import libpdalpython From dc8b1b590f0e72fb36d29193da0681a811508f20 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 5 Jan 2024 09:11:52 -0600 Subject: [PATCH 20/59] Cleanup GIL stuff (#157) * executeStream already does GIL release, we don't need to do it. pybind11 does GIL acquire with every method call so we don't need to be doing that either * wip * put back acq * wip * wip2 * wip3 * wip4 * wip4 * comment why we need gil_scope_acquire for getExecutor --- pdal/libpdalpython.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pdal/libpdalpython.cpp b/pdal/libpdalpython.cpp index a7b5147a..7bd526a0 100644 --- a/pdal/libpdalpython.cpp +++ b/pdal/libpdalpython.cpp @@ -30,7 +30,6 @@ namespace pdal { }; std::vector getDrivers() { - py::gil_scoped_acquire acquire; std::vector drivers; pdal::StageFactory f(false); @@ -59,7 +58,6 @@ namespace pdal { }; py::object getOptions() { - py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); py::dict stageOptions; @@ -94,7 +92,6 @@ namespace pdal { }; std::vector getDimensions() { - py::gil_scoped_acquire acquire; py::object np = py::module_::import("numpy"); py::object dtype = np.attr("dtype"); std::vector dims; @@ -112,13 +109,11 @@ namespace pdal { std::string getReaderDriver(std::filesystem::path const& p) { - py::gil_scoped_acquire acquire; return StageFactory::inferReaderDriver(p.string()); } std::string getWriterDriver(std::filesystem::path const& p) { - py::gil_scoped_acquire acquire; return StageFactory::inferWriterDriver(p.string()); } @@ -142,7 +137,6 @@ namespace pdal { } py::object getMetadata() { - py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); std::stringstream strm; @@ -190,7 +184,6 @@ namespace pdal { } void setInputs(std::vector ndarrays) { - py::gil_scoped_acquire acquire; _inputs.clear(); for (const auto& ndarray: ndarrays) { PyArrayObject* ndarray_ptr = (PyArrayObject*)ndarray.ptr(); @@ -209,7 +202,6 @@ namespace pdal { std::string getSrsWKT2() { return getExecutor()->getSrsWKT2(); } py::object getQuickInfo() { - py::gil_scoped_acquire acquire; py::object json = py::module_::import("json"); std::string response; @@ -275,6 +267,10 @@ namespace pdal { void delExecutor() { _executor.reset(); } PipelineExecutor* getExecutor() { + // We need to acquire the GIL before we create the executor + // because this method does Python init stuff but pybind11 doesn't + // automatically encapsulate it with a gil_scoped_acquire like it + // does for all of the other methods it knows about py::gil_scoped_acquire acquire; if (!_executor) _executor.reset(new PipelineExecutor(getJson(), _inputs, _loglevel)); From 52f94b0479e67f6fe51550f883d0edf2aa338a39 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 24 Apr 2024 13:09:33 -0500 Subject: [PATCH 21/59] Update build to use scikit-build-core (#165) * modernize with scikit-build-core * update metata * CI churn * add convenience for printing info about the library and getting the pdal-plugin-path * Release build type * Test --no-deps --no-build-isolation * release stuff * add note about needing PDAL base library to PyPI README * dynamic version, pin scikit-build-core > 0.9 * remove cruft --- .github/environment.yml | 2 +- .github/workflows/build.yml | 44 +++---------- .github/workflows/release.yml | 44 +++++++++++++ CMakeLists.txt | 12 ++-- README.rst | 6 +- pyproject.toml | 69 +++++++++++++++++++- setup.py => setup.py.off | 0 {pdal => src/pdal}/PyArray.cpp | 0 {pdal => src/pdal}/PyArray.hpp | 0 {pdal => src/pdal}/PyDimension.hpp | 0 {pdal => src/pdal}/PyPipeline.cpp | 0 {pdal => src/pdal}/PyPipeline.hpp | 0 {pdal => src/pdal}/StreamableExecutor.cpp | 0 {pdal => src/pdal}/StreamableExecutor.hpp | 0 {pdal => src/pdal}/__init__.py | 2 +- src/pdal/__main__.py | 77 +++++++++++++++++++++++ {pdal => src/pdal}/drivers.py | 0 {pdal => src/pdal}/libpdalpython.cpp | 0 {pdal => src/pdal}/pipeline.py | 0 19 files changed, 208 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/release.yml rename setup.py => setup.py.off (100%) rename {pdal => src/pdal}/PyArray.cpp (100%) rename {pdal => src/pdal}/PyArray.hpp (100%) rename {pdal => src/pdal}/PyDimension.hpp (100%) rename {pdal => src/pdal}/PyPipeline.cpp (100%) rename {pdal => src/pdal}/PyPipeline.hpp (100%) rename {pdal => src/pdal}/StreamableExecutor.cpp (100%) rename {pdal => src/pdal}/StreamableExecutor.hpp (100%) rename {pdal => src/pdal}/__init__.py (94%) create mode 100644 src/pdal/__main__.py rename {pdal => src/pdal}/drivers.py (100%) rename {pdal => src/pdal}/libpdalpython.cpp (100%) rename {pdal => src/pdal}/pipeline.py (100%) diff --git a/.github/environment.yml b/.github/environment.yml index c12722d8..5a4905c7 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -2,7 +2,7 @@ name: testenv channels: - conda-forge dependencies: - - scikit-build + - scikit-build-core - numpy - compilers - pybind11 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc5c8969..65f01ff3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,60 +35,30 @@ jobs: with: repository: PDAL/python-plugins path: ./plugins + ref: main - name: Setup micromamba uses: conda-incubator/setup-miniconda@v3 with: miniforge-variant: Mambaforge miniforge-version: latest + python-version: ${{ matrix.python-version }} use-mamba: true auto-update-conda: true environment-file: .github/environment.yml - name: Install python-pdal - run: pip install . + run: | + pip install -vv . --no-deps --no-build-isolation - name: Install python-pdal-plugins working-directory: ./plugins - run: pip install . + run: pip install -vv . --no-deps --no-build-isolation - name: Test run: | - export PDAL_DRIVER_PATH=$(python -c "import os, skbuild; print(os.path.join('plugins', skbuild.constants.SKBUILD_DIR(), 'cmake-build'))") + export PDAL_DRIVER_PATH=$(python -m pdal --pdal-plugin-path) + echo $PDAL_DRIVER_PATH pdal --drivers --debug py.test -v test/ - - name: Build source distribution - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' - run: python setup.py sdist - - - name: Upload distribution(s) - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.os }}-py${{ matrix.python-version }} - path: ./dist/* - - publish: - runs-on: ubuntu-latest - needs: [build] - - steps: - - name: Download distributions - uses: actions/download-artifact@v3 - with: - path: ./artifacts - - - name: Move artifacts to dist - run: | - mkdir dist - find ./artifacts -type f -exec mv {} ./dist \; - tree ./dist - - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 - if: github.event_name == 'release' && github.event.action == 'published' - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} - packages_dir: ./dist - verbose: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..6b55754a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + release: + types: + - published + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + pypi-publish: + name: Upload release to PyPI + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/p/pdal-plugins + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@v3 + - name: Setup micromamba + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + python-version: ${{ matrix.python-version }} + auto-update-conda: true + environment-file: .github/environment.yml + + - name: Install dependencies + shell: bash -l {0} + run: | + python -m pip install build pipx twine + pipx run build --sdist + + - name: Publish package distributions to PyPI + if: github.event_name == 'release' && github.event.action == 'published' + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CMakeLists.txt b/CMakeLists.txt index b45f0c35..8c2860ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.11.0) -project(pdal-python) +project(pdal-python VERSION) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -23,17 +23,17 @@ endif() find_package(Python3 COMPONENTS Interpreter ${DEVELOPMENT_COMPONENT} NumPy REQUIRED) # find PDAL. Require 2.1+ -find_package(PDAL 2.5 REQUIRED) +find_package(PDAL 2.6 REQUIRED) # find PyBind11 find_package(pybind11 REQUIRED) set(extension "libpdalpython") pybind11_add_module(${extension} MODULE - pdal/PyArray.cpp - pdal/PyPipeline.cpp - pdal/StreamableExecutor.cpp - pdal/libpdalpython.cpp + src/pdal/PyArray.cpp + src/pdal/PyPipeline.cpp + src/pdal/StreamableExecutor.cpp + src/pdal/libpdalpython.cpp ) target_include_directories(${extension} PRIVATE ${Python3_NumPy_INCLUDE_DIRS}) target_link_libraries(${extension} PRIVATE ${PDAL_LIBRARIES}) diff --git a/README.rst b/README.rst index 1a2d9de6..77915dda 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,8 @@ Additionally, you can use it to fetch `schema`_ and `metadata`_ from PDAL operat Installation -------------------------------------------------------------------------------- +**Note** The PDAL Python bindings require the PDAL base library installed. Source code can be found at https://pdal.io and `GitHub `__. + PyPI ................................................................................ @@ -307,8 +309,8 @@ USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into ti Requirements ================================================================================ -* PDAL 2.5+ +* PDAL 2.6+ * Python >=3.9 * Pybind11 (eg :code:`pip install pybind11[global]`) * Numpy (eg :code:`pip install numpy`) -* scikit-build (eg :code:`pip install scikit-build`) +* scikit-build-core (eg :code:`pip install scikit-build-core`) diff --git a/pyproject.toml b/pyproject.toml index db01ea67..419a8b08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,69 @@ +[project] +name = "pdal" +description = "Point cloud data processing" +readme = "README.rst" +requires-python = ">=3.9" +license = {file = "LICENSE.txt"} +keywords = ["point", "cloud", "spatial"] +authors = [ + {email = "howard@hobu.co"}, + {name = "Howard Butler"} +] +maintainers = [ + {name = "Howard Butler", email = "howard@hobu.co"} +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: GIS", +] + +dependencies = [ + "numpy" +] + +dynamic = ["version"] + +[project.optional-dependencies] +test = [ + "pandas", + "meshio" +] + +[tool.setuptools] +package-dir = {"" = "src"} +zip-safe = false + +[project.urls] +homepage = "https://pdal.io" +documentation = "https://pdal.io" +repository = "https://github.com/PDAL/Python" +changelog = "https://github.com/PDAL/python/blob/main/README.rst" + [build-system] -requires = ["scikit-build", "cmake>=3.11", "ninja", "numpy", "pybind11[global]"] +requires = ["scikit-build-core >= 0.9", "numpy", "pybind11[global]"] +build-backend = "scikit_build_core.build" + + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +sdist.exclude = [".github"] +sdist.cmake = true +cmake.build-type = "Release" +sdist.include = [ + "src", + "CMakeLists.txt" +] +cmake.verbose = false +logging.level = "ERROR" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "src/pdal/__init__.py" diff --git a/setup.py b/setup.py.off similarity index 100% rename from setup.py rename to setup.py.off diff --git a/pdal/PyArray.cpp b/src/pdal/PyArray.cpp similarity index 100% rename from pdal/PyArray.cpp rename to src/pdal/PyArray.cpp diff --git a/pdal/PyArray.hpp b/src/pdal/PyArray.hpp similarity index 100% rename from pdal/PyArray.hpp rename to src/pdal/PyArray.hpp diff --git a/pdal/PyDimension.hpp b/src/pdal/PyDimension.hpp similarity index 100% rename from pdal/PyDimension.hpp rename to src/pdal/PyDimension.hpp diff --git a/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp similarity index 100% rename from pdal/PyPipeline.cpp rename to src/pdal/PyPipeline.cpp diff --git a/pdal/PyPipeline.hpp b/src/pdal/PyPipeline.hpp similarity index 100% rename from pdal/PyPipeline.hpp rename to src/pdal/PyPipeline.hpp diff --git a/pdal/StreamableExecutor.cpp b/src/pdal/StreamableExecutor.cpp similarity index 100% rename from pdal/StreamableExecutor.cpp rename to src/pdal/StreamableExecutor.cpp diff --git a/pdal/StreamableExecutor.hpp b/src/pdal/StreamableExecutor.hpp similarity index 100% rename from pdal/StreamableExecutor.hpp rename to src/pdal/StreamableExecutor.hpp diff --git a/pdal/__init__.py b/src/pdal/__init__.py similarity index 94% rename from pdal/__init__.py rename to src/pdal/__init__.py index 1f05ff3d..7ba4fad7 100644 --- a/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ -__version__ = "3.3.0" __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] +__version__ = '3.4.0' from . import libpdalpython from .drivers import inject_pdal_drivers diff --git a/src/pdal/__main__.py b/src/pdal/__main__.py new file mode 100644 index 00000000..f45f3151 --- /dev/null +++ b/src/pdal/__main__.py @@ -0,0 +1,77 @@ +import sys +import os +import pathlib + +import distutils.ccompiler +so_ext = distutils.ccompiler.new_compiler().shared_lib_extension +import sysconfig + +import argparse + +import pdal + +from . import __version__ + +__all__ = ["main"] + + +def __dir__() -> list[str]: + return __all__ + + +def print_driver_path(args): + print (os.environ['PDAL_DRIVER_PATH']) + +def print_plugin_path(args): + purelib = sysconfig.get_paths()["purelib"]+os.path.sep+"pdal" + + if sys.platform == "linux" or sys.platform == "linux2": + suffix = 'so' + purelib = purelib + os.path.sep + "pdal" + elif sys.platform == "darwin": + suffix = 'dylib' + purelib = purelib + os.path.sep + "pdal" + elif sys.platform == "win32": + suffix = 'dll' + purelib = purelib + os.path.sep + "bin" + + for f in pathlib.Path(purelib).glob(f'*.{suffix}'): + if 'pdal' in str(f.name): + if 'numpy' in str(f.name) or 'python' in str(f.name): + print (purelib) + return # we are done + +def print_version(args): + info = pdal.drivers.libpdalpython.getInfo() + pdal_version = info.version + plugin = info.plugin + debug = info.debug + + line = '----------------------------------------------------------------------------------------------------------------------------\n' + version = f'PDAL version {pdal_version}\nPython bindings version {__version__}\n' + plugin = f"Environment-set PDAL_DRIVER_PATH: {os.environ['PDAL_DRIVER_PATH']}" + output = f'{line}{version}{plugin}\n{line}\n{debug}' + print (output) + + +def main() -> None: + header = f"PDAL Python bindings {__version__} on Python {sys.version}" + + parser = argparse.ArgumentParser(description=header) + parser.add_argument('--pdal-driver-path', action='store_true', + help='print PDAL_DRIVER_PATH including Python plugin locations') + parser.add_argument('--pdal-plugin-path', action='store_true', + help='print location of PDAL Python plugins') + + args = parser.parse_args() + + if args.pdal_driver_path: + print_driver_path(args) + elif args.pdal_plugin_path: + print_plugin_path(args) + else: + print_version(args) + + +if __name__ == "__main__": + main() diff --git a/pdal/drivers.py b/src/pdal/drivers.py similarity index 100% rename from pdal/drivers.py rename to src/pdal/drivers.py diff --git a/pdal/libpdalpython.cpp b/src/pdal/libpdalpython.cpp similarity index 100% rename from pdal/libpdalpython.cpp rename to src/pdal/libpdalpython.cpp diff --git a/pdal/pipeline.py b/src/pdal/pipeline.py similarity index 100% rename from pdal/pipeline.py rename to src/pdal/pipeline.py From 5e31b65dd55866ca2491875a5b1b782673e0f558 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sun, 28 Apr 2024 09:36:51 -0500 Subject: [PATCH 22/59] Set Numpy floor to 1.22 and NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION (#167) * Set Numpy floor to 1.22 and NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION * set PY_ARRAY_UNIQUE_SYMBOL to PDAL_ARRAY_API and only import it in extension initialization. Guard against using the extension against any PDAL verisions < 2.6 * debug PDAL_DRIVER_PATH issues * bug in --print-plugin-path * only one os.path.sep * concat PDAL_DRIVER_PATH with PDAL_PLUGIN_PATH --- .github/workflows/build.yml | 8 ++++++-- CMakeLists.txt | 4 +++- pyproject.toml | 4 ++-- src/pdal/PyArray.cpp | 3 --- src/pdal/PyArray.hpp | 9 +++++++++ src/pdal/PyPipeline.cpp | 16 +--------------- src/pdal/PyPipeline.hpp | 7 +++++++ src/pdal/StreamableExecutor.cpp | 10 ++++++---- src/pdal/__main__.py | 2 +- src/pdal/libpdalpython.cpp | 18 ++++++++++++++++++ 10 files changed, 53 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65f01ff3..42c82a78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,8 +57,12 @@ jobs: - name: Test run: | - export PDAL_DRIVER_PATH=$(python -m pdal --pdal-plugin-path) - echo $PDAL_DRIVER_PATH + export PDAL_DRIVER_PATH=$(python -m pdal --pdal-driver-path) + export PDAL_PLUGIN_PATH=$(python -m pdal --pdal-plugin-path) + echo "PDAL_DRIVER_PATH $PDAL_DRIVER_PATH" + echo "PDAL_PLUGIN_PATH $PDAL_PLUGIN_PATH" + export PDAL_DRIVER_PATH=$PDAL_PLUGIN_PATH:$PDAL_DRIVER_PATH + python -m pdal pdal --drivers --debug py.test -v test/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c2860ed..e22cd1a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.11.0) -project(pdal-python VERSION) +project(pdal-python VERSION ${SKBUILD_PROJECT_VERSION} + DESCRIPTION "PDAL Python bindings" + HOMEPAGE_URL "https://github.com/PDAL/Python") set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) diff --git a/pyproject.toml b/pyproject.toml index 419a8b08..eca080ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ - "numpy" + "numpy >= 1.22" ] dynamic = ["version"] @@ -48,7 +48,7 @@ repository = "https://github.com/PDAL/Python" changelog = "https://github.com/PDAL/python/blob/main/README.rst" [build-system] -requires = ["scikit-build-core >= 0.9", "numpy", "pybind11[global]"] +requires = ["scikit-build-core >= 0.9", "numpy >= 1.22", "pybind11[global]"] build-backend = "scikit_build_core.build" diff --git a/src/pdal/PyArray.cpp b/src/pdal/PyArray.cpp index 04c424bd..66b6bd45 100644 --- a/src/pdal/PyArray.cpp +++ b/src/pdal/PyArray.cpp @@ -92,9 +92,6 @@ std::string toString(PyObject *pname) Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); - Py_XINCREF(array); PyArray_Descr *dtype = PyArray_DTYPE(m_array); diff --git a/src/pdal/PyArray.hpp b/src/pdal/PyArray.hpp index f0401f00..e2da961f 100644 --- a/src/pdal/PyArray.hpp +++ b/src/pdal/PyArray.hpp @@ -35,8 +35,17 @@ #pragma once #include + +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include + #include +#include #include #include diff --git a/src/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp index 35859f06..85ea028a 100644 --- a/src/pdal/PyPipeline.cpp +++ b/src/pdal/PyPipeline.cpp @@ -211,16 +211,7 @@ std::string PipelineExecutor::getQuickInfo() const void PipelineExecutor::addArrayReaders(std::vector> arrays) { - // Make the symbols in pdal_base global so that they're accessible - // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, - // which means that without this, symbols in libpdal_base aren't available - // for resolution of symbols on future runtime linking. This is an issue - // on Alpine and other Linux variants that don't use UNIQUE symbols - // for C++ template statics only. Without this, you end up with multiple - // copies of template statics. -#ifndef _WIN32 - ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); -#endif + if (arrays.empty()) return; @@ -320,8 +311,6 @@ PyObject* buildNumpyDescriptor(PointLayoutPtr layout) PyArrayObject* viewToNumpyArray(PointViewPtr view) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); PyObject* dtype_dict = buildNumpyDescriptor(view->layout()); PyArray_Descr *dtype = nullptr; @@ -344,9 +333,6 @@ PyArrayObject* viewToNumpyArray(PointViewPtr view) PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh) { - if (_import_array() < 0) - throw pdal_error("Could not import numpy.core.multiarray."); - // Build up a numpy dtype dictionary // // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', diff --git a/src/pdal/PyPipeline.hpp b/src/pdal/PyPipeline.hpp index c4e316c6..c32abcfe 100644 --- a/src/pdal/PyPipeline.hpp +++ b/src/pdal/PyPipeline.hpp @@ -35,6 +35,13 @@ #pragma once #include + +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include namespace pdal diff --git a/src/pdal/StreamableExecutor.cpp b/src/pdal/StreamableExecutor.cpp index ef9950e2..9f5b4b8b 100644 --- a/src/pdal/StreamableExecutor.cpp +++ b/src/pdal/StreamableExecutor.cpp @@ -35,6 +35,9 @@ #include "PyPipeline.hpp" #include "StreamableExecutor.hpp" +#define NO_IMPORT_ARRAY +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + #include #include @@ -67,8 +70,7 @@ void PythonPointTable::finalize() // create dtype auto gil = PyGILState_Ensure(); - if (_import_array() < 0) - std::cerr << "Could not import array!\n"; + PyObject *dtype_dict = buildNumpyDescriptor(&m_layout); if (PyArray_DescrConverter(dtype_dict, &m_dtype) == NPY_FAIL) throw pdal_error("Unable to create numpy dtype"); @@ -102,8 +104,8 @@ void PythonPointTable::py_resizeArray(point_count_t np) { if (src_idx != dest_idx) { - PyObject* src_item = PyArray_GETITEM(m_curArray, PyArray_GETPTR1(m_curArray, src_idx)); - PyArray_SETITEM(m_curArray, PyArray_GETPTR1(m_curArray, dest_idx), src_item); + PyObject* src_item = PyArray_GETITEM(m_curArray, (const char*) PyArray_GETPTR1(m_curArray, src_idx)); + PyArray_SETITEM(m_curArray, (char*) PyArray_GETPTR1(m_curArray, dest_idx), src_item); Py_XDECREF(src_item); } dest_idx++; diff --git a/src/pdal/__main__.py b/src/pdal/__main__.py index f45f3151..7fa7d272 100644 --- a/src/pdal/__main__.py +++ b/src/pdal/__main__.py @@ -23,7 +23,7 @@ def print_driver_path(args): print (os.environ['PDAL_DRIVER_PATH']) def print_plugin_path(args): - purelib = sysconfig.get_paths()["purelib"]+os.path.sep+"pdal" + purelib = sysconfig.get_paths()["purelib"] if sys.platform == "linux" or sys.platform == "linux2": suffix = 'so' diff --git a/src/pdal/libpdalpython.cpp b/src/pdal/libpdalpython.cpp index 7bd526a0..229f928d 100644 --- a/src/pdal/libpdalpython.cpp +++ b/src/pdal/libpdalpython.cpp @@ -7,6 +7,13 @@ #include #include +#define NPY_TARGET_VERSION NPY_1_22_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION + +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API + +#include + #include "PyArray.hpp" #include "PyDimension.hpp" #include "PyPipeline.hpp" @@ -283,8 +290,12 @@ namespace pdal { int _loglevel; }; + + PYBIND11_MODULE(libpdalpython, m) { + _import_array(); + py::class_(m, "PipelineIterator") .def("__iter__", [](PipelineIterator &it) -> PipelineIterator& { return it; }) .def("__next__", &PipelineIterator::executeNext) @@ -294,6 +305,7 @@ namespace pdal { .def_property_readonly("pipeline", &PipelineIterator::getPipeline) .def_property_readonly("metadata", &PipelineIterator::getMetadata); + py::class_(m, "Pipeline") .def(py::init<>()) .def("execute", &Pipeline::execute) @@ -319,6 +331,12 @@ namespace pdal { m.def("getDimensions", &getDimensions); m.def("infer_reader_driver", &getReaderDriver); m.def("infer_writer_driver", &getWriterDriver); + + if (pdal::Config::versionMajor() < 2) + throw pybind11::import_error("PDAL version must be >= 2.6"); + + if (pdal::Config::versionMajor() == 2 && pdal::Config::versionMinor() < 6) + throw pybind11::import_error("PDAL version must be >= 2.6"); }; }; // namespace pdal From f4bbc6a64cd37966de1b62aef7dbcf6fc3799fb4 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 29 Apr 2024 15:18:16 -0500 Subject: [PATCH 23/59] add note about Numpy >= 1.22 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 77915dda..0132ec84 100644 --- a/README.rst +++ b/README.rst @@ -312,5 +312,5 @@ Requirements * PDAL 2.6+ * Python >=3.9 * Pybind11 (eg :code:`pip install pybind11[global]`) -* Numpy (eg :code:`pip install numpy`) +* Numpy >= 1.22 (eg :code:`pip install numpy`) * scikit-build-core (eg :code:`pip install scikit-build-core`) From f369e7502376b539951931c9c28fe02526a7b8b8 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 29 Apr 2024 16:31:01 -0500 Subject: [PATCH 24/59] bump 3.4.1 for release churn --- .github/workflows/release.yml | 1 + src/pdal/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6b55754a..eff5ecc8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,3 +42,4 @@ jobs: - name: Publish package distributions to PyPI if: github.event_name == 'release' && github.event.action == 'published' uses: pypa/gh-action-pypi-publish@release/v1 + diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index 7ba4fad7..09bd31b1 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.4.0' +__version__ = '3.4.1' from . import libpdalpython from .drivers import inject_pdal_drivers From ac8b981a36519f355325cc2c1f1fa983229c9593 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 14 May 2024 20:21:26 -0500 Subject: [PATCH 25/59] distutils doesnt exist on 3.12 (#168) --- src/pdal/__main__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pdal/__main__.py b/src/pdal/__main__.py index 7fa7d272..b0da6098 100644 --- a/src/pdal/__main__.py +++ b/src/pdal/__main__.py @@ -2,8 +2,6 @@ import os import pathlib -import distutils.ccompiler -so_ext = distutils.ccompiler.new_compiler().shared_lib_extension import sysconfig import argparse From 7afdf3ca045a0df26f81eae4e189fef546bbafd4 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 14 May 2024 20:38:37 -0500 Subject: [PATCH 26/59] bump version number to 3.4.2 --- src/pdal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index 09bd31b1..b0e7d699 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.4.1' +__version__ = '3.4.2' from . import libpdalpython from .drivers import inject_pdal_drivers From 6245b00de6b5e8910f7ca2c7a3db52af5fee85ea Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sun, 19 May 2024 21:50:01 -0500 Subject: [PATCH 27/59] check for PDAL_DRIVER_PATH in the environment before accessing --- src/pdal/__init__.py | 2 +- src/pdal/__main__.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index b0e7d699..5c2feabb 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.4.2' +__version__ = '3.4.3' from . import libpdalpython from .drivers import inject_pdal_drivers diff --git a/src/pdal/__main__.py b/src/pdal/__main__.py index b0da6098..8c85020c 100644 --- a/src/pdal/__main__.py +++ b/src/pdal/__main__.py @@ -18,7 +18,8 @@ def __dir__() -> list[str]: def print_driver_path(args): - print (os.environ['PDAL_DRIVER_PATH']) + if 'PDAL_DRIVER_PATH' in os.environ: + print (os.environ['PDAL_DRIVER_PATH']) def print_plugin_path(args): purelib = sysconfig.get_paths()["purelib"] From c03f98739328438a8bee76696af0fc6d1969e155 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sun, 19 May 2024 21:56:24 -0500 Subject: [PATCH 28/59] bump up checkout action version --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eff5ecc8..17ab0125 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: fail-fast: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup micromamba uses: conda-incubator/setup-miniconda@v3 with: From c70fd363926f49b11b6e75ddfe79fc844305a1c3 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 17 Jun 2024 17:16:36 -0500 Subject: [PATCH 29/59] use dtype's fields PyObject* now (#170) * use dtype's fields PyObject* now * bump version for this to 3.4.4 --- src/pdal/PyArray.cpp | 8 +++++--- src/pdal/__init__.py | 2 +- src/pdal/__main__.py | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pdal/PyArray.cpp b/src/pdal/PyArray.cpp index 66b6bd45..d3521173 100644 --- a/src/pdal/PyArray.cpp +++ b/src/pdal/PyArray.cpp @@ -97,9 +97,11 @@ Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) PyArray_Descr *dtype = PyArray_DTYPE(m_array); npy_intp ndims = PyArray_NDIM(m_array); npy_intp *shape = PyArray_SHAPE(m_array); - int numFields = (dtype->fields == Py_None) ? + + PyObject* fields = PyDataType_FIELDS(dtype); + int numFields = (fields == Py_None) ? 0 : - static_cast(PyDict_Size(dtype->fields)); + static_cast(PyDict_Size(fields)); int xyz = 0; if (numFields == 0) @@ -110,7 +112,7 @@ Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) } else { - PyObject *names_dict = dtype->fields; + PyObject *names_dict = fields; PyObject *names = PyDict_Keys(names_dict); PyObject *values = PyDict_Values(names_dict); if (!names || !values) diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index 5c2feabb..41b36121 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.4.3' +__version__ = '3.4.4' from . import libpdalpython from .drivers import inject_pdal_drivers diff --git a/src/pdal/__main__.py b/src/pdal/__main__.py index 8c85020c..4569e522 100644 --- a/src/pdal/__main__.py +++ b/src/pdal/__main__.py @@ -48,7 +48,10 @@ def print_version(args): line = '----------------------------------------------------------------------------------------------------------------------------\n' version = f'PDAL version {pdal_version}\nPython bindings version {__version__}\n' - plugin = f"Environment-set PDAL_DRIVER_PATH: {os.environ['PDAL_DRIVER_PATH']}" + driver_path = 'PDAL_DRIVER_PATH not set!' + if 'PDAL_DRIVER_PATH' in os.environ: + driver_path = os.environ['PDAL_DRIVER_PATH'] + plugin = f"Environment-set PDAL_DRIVER_PATH: {driver_path}" output = f'{line}{version}{plugin}\n{line}\n{debug}' print (output) From 3d16417044506ab04deb9e4247fcaf25019da27c Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 18 Jun 2024 10:03:49 -0500 Subject: [PATCH 30/59] handle ABI < 2 (#171) --- src/pdal/PyArray.cpp | 4 ++++ src/pdal/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pdal/PyArray.cpp b/src/pdal/PyArray.cpp index d3521173..55f87b5d 100644 --- a/src/pdal/PyArray.cpp +++ b/src/pdal/PyArray.cpp @@ -90,6 +90,10 @@ std::string toString(PyObject *pname) } // unnamed namespace +#if NPY_ABI_VERSION < 0x02000000 + #define PyDataType_FIELDS(descr) ((descr)->fields) +#endif + Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) { Py_XINCREF(array); diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index 41b36121..a312f568 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.4.4' +__version__ = '3.4.5' from . import libpdalpython from .drivers import inject_pdal_drivers From 4205f3dac0de7b4cb276f262159b7ffe1fb46cde Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 18 Jun 2024 12:39:15 -0500 Subject: [PATCH 31/59] parameterize numpy testing (#172) * parameterize numpy * typo * specify numpy * exclude 3.12 + 1.22 * typo * more precise exclude * ID the job * increase to 1.24 --- .github/workflows/build.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42c82a78..47180b8d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ defaults: jobs: build: - name: Build ${{ matrix.os }} py${{ matrix.python-version }} + name: ${{ matrix.os }} py${{ matrix.python-version }} numpy ${{ matrix.numpy-version }} runs-on: ${{ matrix.os }} strategy: @@ -25,6 +25,10 @@ jobs: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ['3.9', '3.10', '3.11', '3.12'] + numpy-version: ['1.24', '2.0'] + exclude: + - python-version: '3.12' + numpy-version: '1.24' steps: - name: Check out python-pdal @@ -47,6 +51,10 @@ jobs: auto-update-conda: true environment-file: .github/environment.yml + - name: Install numpy ${{ matrix.numpy-version }} + run: | + mamba install -y numpy=${{ matrix.numpy-version }} + - name: Install python-pdal run: | pip install -vv . --no-deps --no-build-isolation From b73f7fdb7858407b69cca8d4e5eeb4aacddad214 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 4 Oct 2024 15:00:12 -0500 Subject: [PATCH 32/59] update readme to include developer build instructions (#180) --- README.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 0132ec84..9b6e123f 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,21 @@ PDAL Python support is installable via PyPI: pip install PDAL + +Developers can control many settings including debug builds and where the libraries are installed +using `scikit-build-core `_ settings: + +.. code-block:: + + python -m pip install \ + -Cbuild-dir=build \ + -e \ + . \ + --config-settings=cmake.build-type="Debug" \ + -vv \ + --no-deps \ + --no-build-isolation + GitHub ................................................................................ @@ -168,7 +183,7 @@ PDAL and Python: print(len(intensity)) # 704 points # Now use pdal to clamp points that have intensity 100 <= v < 300 - pipeline = pdal.Filter.range(limits="Intensity[100:300)").pipeline(intensity) + pipeline = pdal.Filter.expression(expression="Intensity >= 100 && Intensity < 300").pipeline(intensity) print(pipeline.execute()) # 387 points clamped = pipeline.arrays[0] @@ -203,7 +218,7 @@ returns an iterator object that yields Numpy arrays of up to ``chunk_size`` size .. code-block:: python import pdal - pipeline = pdal.Reader("test/data/autzen-utm.las") | pdal.Filter.range(limits="Intensity[80:120)") + pipeline = pdal.Reader("test/data/autzen-utm.las") | pdal.Filter.expression(expression="Intensity > 80 && Intensity < 120)") for array in pipeline.iterator(chunk_size=500): print(len(array)) # or to concatenate all arrays into one From b83d78dcc728fc3ccc190df5aefcdf65c7a6a814 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 4 Oct 2024 15:00:37 -0500 Subject: [PATCH 33/59] update deprecated miniforge-variant to Miniforge --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47180b8d..49934cbd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: - name: Setup micromamba uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge + miniforge-variant: Miniforge3 miniforge-version: latest python-version: ${{ matrix.python-version }} use-mamba: true From f8583508cb911cacbdbab065beaf3f41009c7ef4 Mon Sep 17 00:00:00 2001 From: James Ford Date: Sat, 5 Oct 2024 09:04:23 +1300 Subject: [PATCH 34/59] Add GeoDataFrame support to Pipeline (#173) * Added GeoDataFrame support to pipeline.py Added basic GeoPandas GeoDataFrame support. If GeoPandas is installed users can read an array from an executed pipeline and return a GeoDataFrame, with optional arguments for XY vs XYZ point and CRS. DataFrames passed to the Pipeline constructor will drop the "geometry" column if present. * Update test_pipeline.py Added test for GeoDataFrames * add geopandas to environment reqs --------- Co-authored-by: Howard Butler --- .github/environment.yml | 2 +- src/pdal/pipeline.py | 22 +++++++++++++-- test/test_pipeline.py | 59 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index 5a4905c7..76875d6b 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -9,4 +9,4 @@ dependencies: - pdal - pytest - meshio - - pandas + - geopandas diff --git a/src/pdal/pipeline.py b/src/pdal/pipeline.py index c13a6d2c..37d98163 100644 --- a/src/pdal/pipeline.py +++ b/src/pdal/pipeline.py @@ -17,6 +17,11 @@ except ModuleNotFoundError: # pragma: no cover DataFrame = None +try: + from geopandas import GeoDataFrame, points_from_xy +except ModuleNotFoundError: # pragma: no cover + GeoDataFrame = points_from_xy = None + from . import drivers, libpdalpython LogLevelToPDAL = { @@ -45,7 +50,7 @@ def __init__( # Convert our data frames to Numpy Structured Arrays if dataframes: - arrays = [df.to_records() for df in dataframes] + arrays = [df.to_records() if not "geometry" in df.columns else df.drop(columns=["geometry"]).to_records() for df in dataframes] super().__init__() self._stages: List[Stage] = [] @@ -124,13 +129,26 @@ def get_meshio(self, idx: int) -> Optional[Mesh]: [("triangle", np.stack((mesh["A"], mesh["B"], mesh["C"]), 1))], ) - def get_dataframe(self, idx: int) -> Optional[DataFrame]: if DataFrame is None: raise RuntimeError("Pandas support requires Pandas to be installed") return DataFrame(self.arrays[idx]) + def get_geodataframe(self, idx: int, xyz: bool=False, crs: Any=None) -> Optional[GeoDataFrame]: + if GeoDataFrame is None: + raise RuntimeError("GeoPandas support requires GeoPandas to be installed") + df = DataFrame(self.arrays[idx]) + coords = [df["X"], df["Y"], df["Z"]] if xyz else [df["X"], df["Y"]] + geometry = points_from_xy(*coords) + gdf = GeoDataFrame( + df, + geometry=geometry, + crs=crs, + ) + df = coords = geometry = None + return gdf + def _get_json(self) -> str: return self.toJSON() diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 5a40b58e..c0c417a8 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -541,6 +541,65 @@ def test_load(self): assert data["Intensity"].sum() == 57684 +class TestGeoDataFrame: + + @pytest.mark.skipif( + not pdal.pipeline.GeoDataFrame, + reason="geopandas is not available", + ) + def test_fetch(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + record_count = p.arrays[0].shape[0] + dimension_count = len(p.arrays[0].dtype) + gdf = p.get_geodataframe(0) + gdf_xyz = p.get_geodataframe(0, xyz=True) + gdf_crs = p.get_geodataframe(0, crs="EPSG:4326") + assert len(gdf) == record_count + assert len(gdf.columns) == dimension_count + 1 + assert isinstance(gdf, pdal.pipeline.GeoDataFrame) + assert gdf.geometry.is_valid.all() + assert not gdf.geometry.is_empty.any() + assert gdf.crs is None + assert gdf.geometry.z.isna().all() + assert not gdf_xyz.geometry.z.isna().any() + assert gdf_crs.crs.srs == "EPSG:4326" + + @pytest.mark.skipif( + not pdal.pipeline.GeoDataFrame, + reason="geopandas is not available", + ) + def test_load(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")) + p = r.pipeline() + p.execute() + data = p.arrays[0] + gdf = pdal.pipeline.GeoDataFrame( + data, + geometry=pdal.pipeline.points_from_xy(data["X"], data["Y"], data["Z"]) + ) + dataframes = [gdf, gdf, gdf] + filter_intensity = """{ + "pipeline":[ + { + "type":"filters.range", + "limits":"Intensity[100:300)" + } + ] + }""" + p = pdal.Pipeline(filter_intensity, dataframes = dataframes) + p.execute() + arrays = p.arrays + assert len(arrays) == 3 + + # We copied the array three times. Sum the Intensity values + # post filtering to see if we had our intended effect + for data in arrays: + assert len(data) == 387 + assert data["Intensity"].sum() == 57684 + + class TestPipelineIterator: @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_non_streamable(self, filename): From fdabd27683eece4747ada3d2c951500a5dd3d6a1 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 29 Oct 2024 11:15:12 -0500 Subject: [PATCH 35/59] add Python 3.13 to build matrix and drop 3.9 (#181) * add Python 3.13 to build matrix and drop 3.9 * adjust matrix * adjust matrix 2 --- .github/workflows/build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 49934cbd..b460c7b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,11 +24,13 @@ jobs: fail-fast: true matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.9', '3.10', '3.11', '3.12'] - numpy-version: ['1.24', '2.0'] + python-version: ['3.10', '3.11', '3.12', '3.13'] + numpy-version: ['1.24', '2.1'] exclude: - python-version: '3.12' numpy-version: '1.24' + - python-version: '3.13' + numpy-version: '1.24' steps: - name: Check out python-pdal From f2fdc7c48550f7f82972da2cc9e7d24fb91bb551 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 15 Nov 2024 13:07:42 -0600 Subject: [PATCH 36/59] align minimum CMake version to 3.16 to align with GDAL and PDAL (#183) --- CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e22cd1a1..61610cfb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.11.0) +cmake_minimum_required(VERSION 3.16.0) project(pdal-python VERSION ${SKBUILD_PROJECT_VERSION} DESCRIPTION "PDAL Python bindings" HOMEPAGE_URL "https://github.com/PDAL/Python") @@ -6,7 +6,6 @@ project(pdal-python VERSION ${SKBUILD_PROJECT_VERSION} set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_BUILD_TYPE "Release") # Python-finding settings set(Python3_FIND_STRATEGY "LOCATION") @@ -25,7 +24,7 @@ endif() find_package(Python3 COMPONENTS Interpreter ${DEVELOPMENT_COMPONENT} NumPy REQUIRED) # find PDAL. Require 2.1+ -find_package(PDAL 2.6 REQUIRED) +find_package(PDAL 2.7 REQUIRED) # find PyBind11 find_package(pybind11 REQUIRED) From 2d035abdae14538a4314cfe519c6c05f078ae246 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 19 Nov 2024 15:34:07 -0600 Subject: [PATCH 37/59] Support limiting dimensions with execute() and executeStreaming() (#184) --- src/pdal/PyArray.cpp | 5 +++-- src/pdal/PyArray.hpp | 3 ++- src/pdal/PyPipeline.cpp | 15 +++++++++++++-- src/pdal/PyPipeline.hpp | 4 ++-- src/pdal/StreamableExecutor.cpp | 8 +++++++- src/pdal/StreamableExecutor.hpp | 3 ++- src/pdal/libpdalpython.cpp | 25 ++++++++++++------------- test/test_pipeline.py | 23 +++++++++++++++++++++++ 8 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/pdal/PyArray.cpp b/src/pdal/PyArray.cpp index 55f87b5d..0dc875d9 100644 --- a/src/pdal/PyArray.cpp +++ b/src/pdal/PyArray.cpp @@ -78,7 +78,7 @@ Dimension::Type pdalType(int t) return Type::None; } -std::string toString(PyObject *pname) +std::string pyObjectToString(PyObject *pname) { PyObject* r = PyObject_Str(pname); if (!r) @@ -92,6 +92,7 @@ std::string toString(PyObject *pname) #if NPY_ABI_VERSION < 0x02000000 #define PyDataType_FIELDS(descr) ((descr)->fields) + #define PyDataType_NAMES(descr) ((descr)->names) #endif Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) @@ -124,7 +125,7 @@ Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) for (int i = 0; i < numFields; ++i) { - std::string name = toString(PyList_GetItem(names, i)); + std::string name = python::pyObjectToString(PyList_GetItem(names, i)); if (name == "X") xyz |= 1; else if (name == "Y") diff --git a/src/pdal/PyArray.hpp b/src/pdal/PyArray.hpp index e2da961f..d2fb9674 100644 --- a/src/pdal/PyArray.hpp +++ b/src/pdal/PyArray.hpp @@ -55,6 +55,7 @@ namespace pdal namespace python { + class ArrayIter; @@ -87,7 +88,7 @@ class PDAL_DLL Array }; -class ArrayIter +class PDAL_DLL ArrayIter { public: ArrayIter(const ArrayIter&) = delete; diff --git a/src/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp index 85ea028a..b64ef3bb 100644 --- a/src/pdal/PyPipeline.cpp +++ b/src/pdal/PyPipeline.cpp @@ -73,8 +73,13 @@ PipelineExecutor::PipelineExecutor( } -point_count_t PipelineExecutor::execute() +point_count_t PipelineExecutor::execute(pdal::StringList allowedDims) { + if (allowedDims.size()) + { + m_manager.pointTable().layout()->setAllowedDims(allowedDims); + } + point_count_t count = m_manager.execute(); m_executed = true; return count; @@ -92,9 +97,14 @@ std::string PipelineExecutor::getSrsWKT2() const return output; } -point_count_t PipelineExecutor::executeStream(point_count_t streamLimit) +point_count_t PipelineExecutor::executeStream(point_count_t streamLimit, + pdal::StringList allowedDims) { CountPointTable table(streamLimit); + if (allowedDims.size()) + { + pointTable().layout()->setAllowedDims(allowedDims); + } m_manager.executeStream(table); m_executed = true; return table.count(); @@ -272,6 +282,7 @@ PyObject* buildNumpyDescriptor(PointLayoutPtr layout) { return layout->dimOffset(id1) < layout->dimOffset(id2); }; + auto dims = layout->dims(); std::sort(dims.begin(), dims.end(), sortByOffset); diff --git a/src/pdal/PyPipeline.hpp b/src/pdal/PyPipeline.hpp index c32abcfe..5233763f 100644 --- a/src/pdal/PyPipeline.hpp +++ b/src/pdal/PyPipeline.hpp @@ -60,8 +60,8 @@ class PDAL_DLL PipelineExecutor { PipelineExecutor(std::string const& json, std::vector> arrays, int level); virtual ~PipelineExecutor() = default; - point_count_t execute(); - point_count_t executeStream(point_count_t streamLimit); + point_count_t execute(pdal::StringList allowedDims); + point_count_t executeStream(point_count_t streamLimit, pdal::StringList allowedDims); const PointViewSet& views() const; std::string getPipeline() const; diff --git a/src/pdal/StreamableExecutor.cpp b/src/pdal/StreamableExecutor.cpp index 9f5b4b8b..5fa01931 100644 --- a/src/pdal/StreamableExecutor.cpp +++ b/src/pdal/StreamableExecutor.cpp @@ -187,11 +187,17 @@ StreamableExecutor::StreamableExecutor(std::string const& json, std::vector> arrays, int level, point_count_t chunkSize, - int prefetch) + int prefetch, + pdal::StringList allowedDims) : PipelineExecutor(json, arrays, level) , m_table(chunkSize, prefetch) , m_exc(nullptr) { + + if (allowedDims.size()) + { + m_table.layout()->setAllowedDims(allowedDims); + } m_thread.reset(new std::thread([this]() { try { diff --git a/src/pdal/StreamableExecutor.hpp b/src/pdal/StreamableExecutor.hpp index 45d015c3..f565c8ee 100644 --- a/src/pdal/StreamableExecutor.hpp +++ b/src/pdal/StreamableExecutor.hpp @@ -81,7 +81,8 @@ class StreamableExecutor : public PipelineExecutor std::vector> arrays, int level, point_count_t chunkSize, - int prefetch); + int prefetch, + pdal::StringList allowedDim); ~StreamableExecutor(); MetadataNode getMetadata() { return m_table.metadata(); } diff --git a/src/pdal/libpdalpython.cpp b/src/pdal/libpdalpython.cpp index 229f928d..e5fc353a 100644 --- a/src/pdal/libpdalpython.cpp +++ b/src/pdal/libpdalpython.cpp @@ -165,28 +165,27 @@ namespace pdal { class Pipeline { public: - point_count_t execute() { + point_count_t execute(pdal::StringList allowedDims) { point_count_t response(0); { py::gil_scoped_release release; - response = getExecutor()->execute(); + response = getExecutor()->execute(allowedDims); } return response; - } - point_count_t executeStream(point_count_t streamLimit) { + point_count_t executeStream(point_count_t streamLimit, pdal::StringList allowedDims) { point_count_t response(0); { py::gil_scoped_release release; - response = getExecutor()->executeStream(streamLimit); + response = getExecutor()->executeStream(streamLimit, allowedDims); } return response; } - std::unique_ptr iterator(int chunk_size, int prefetch) { + std::unique_ptr iterator(int chunk_size, int prefetch, pdal::StringList allowedDims) { return std::unique_ptr(new PipelineIterator( - getJson(), _inputs, _loglevel, chunk_size, prefetch + getJson(), _inputs, _loglevel, chunk_size, prefetch, allowedDims )); } @@ -308,9 +307,9 @@ namespace pdal { py::class_(m, "Pipeline") .def(py::init<>()) - .def("execute", &Pipeline::execute) - .def("execute_streaming", &Pipeline::executeStream, "chunk_size"_a=10000) - .def("iterator", &Pipeline::iterator, "chunk_size"_a=10000, "prefetch"_a=0) + .def("execute", &Pipeline::execute, py::arg("allowed_dims") =py::list()) + .def("execute_streaming", &Pipeline::executeStream, "chunk_size"_a=10000, py::arg("allowed_dims") =py::list()) + .def("iterator", &Pipeline::iterator, "chunk_size"_a=10000, "prefetch"_a=0, py::arg("allowed_dims") =py::list()) .def_property("inputs", nullptr, &Pipeline::setInputs) .def_property("loglevel", &Pipeline::getLoglevel, &Pipeline::setLogLevel) .def_property_readonly("log", &Pipeline::getLog) @@ -333,10 +332,10 @@ namespace pdal { m.def("infer_writer_driver", &getWriterDriver); if (pdal::Config::versionMajor() < 2) - throw pybind11::import_error("PDAL version must be >= 2.6"); + throw pybind11::import_error("PDAL version must be >= 2.7"); - if (pdal::Config::versionMajor() == 2 && pdal::Config::versionMinor() < 6) - throw pybind11::import_error("PDAL version must be >= 2.6"); + if (pdal::Config::versionMajor() == 2 && pdal::Config::versionMinor() < 7) + throw pybind11::import_error("PDAL version must be >= 2.7"); }; }; // namespace pdal diff --git a/test/test_pipeline.py b/test/test_pipeline.py index c0c417a8..fcec914a 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -82,6 +82,17 @@ def test_execute_streaming(self, filename): count2 = r.execute_streaming(chunk_size=100) assert count == count2 + + @pytest.mark.parametrize("filename", ["range.json", "range.py"]) + def test_subsetstreaming(self, filename): + """Can we fetch a subset of PDAL dimensions as a numpy array while streaming""" + r = get_pipeline(filename) + limit = ['X','Y','Z','Intensity'] + arrays = list(r.iterator(chunk_size=100,allowed_dims=limit)) + assert len(arrays) == 11 + assert len(arrays[0].dtype) == 4 + + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_execute_streaming_non_streamable(self, filename): r = get_pipeline(filename) @@ -113,6 +124,18 @@ def test_array(self, filename): assert a[0][0] == 635619.85 assert a[1064][2] == 456.92 + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) + def test_subsetarray(self, filename): + """Can we fetch a subset of PDAL dimensions as a numpy array""" + r = get_pipeline(filename) + limit = ['X','Y','Z'] + r.execute(allowed_dims=limit) + arrays = r.arrays + assert len(arrays) == 1 + assert len(arrays[0].dtype) == 3 + + + @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_metadata(self, filename): """Can we fetch PDAL metadata""" From 32328ac7050bf517b6961cdc297548a384766b32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o?= <8913464+joaori@users.noreply.github.com> Date: Tue, 17 Dec 2024 20:51:10 +0000 Subject: [PATCH 38/59] Adding support for streaming through the input arrays of a pipeline (#182) * Adding support for streaming the input arrays of a pipeline * Code cleanup based on code review. * Updated documentation to include mention to reading with stream handler. --- README.rst | 98 ++++++++++++++++++++++++ src/pdal/PyArray.cpp | 82 +++++++++++++++----- src/pdal/PyArray.hpp | 19 +++-- src/pdal/PyPipeline.cpp | 14 +++- src/pdal/libpdalpython.cpp | 20 ++++- src/pdal/pipeline.py | 12 ++- test/test_pipeline.py | 148 +++++++++++++++++++++++++++++++++++++ 7 files changed, 356 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index 9b6e123f..cbab9ac1 100644 --- a/README.rst +++ b/README.rst @@ -208,6 +208,104 @@ PDAL and Python: with tiledb.open("clamped") as a: print(a.schema) +Reading using Numpy Arrays as buffers (advanced) +................................................................................ + +It's also possible to treat the Numpy arrays passed to PDAL as buffers that are iteratively populated through +custom python functions during the execution of the pipeline. + +This may be useful in cases where you want the reading of the input data to be handled in a streamable fashion, +like for example: + +* When the total Numpy array data wouldn't fit into memory. +* To initiate execution of a streamable PDAL pipeline while the input data is still being read. + +To enable this mode, you just need to include the python populate function along with each corresponding Numpy array. + +.. code-block:: python + + # Numpy array to be used as buffer + in_buffer = np.zeros(max_chunk_size, dtype=[("X", float), ("Y", float), ("Z", float)]) + + # The function to populate the buffer iteratively + def load_next_chunk() -> int: + """ + Function called by PDAL before reading the data from the buffer. + + IMPORTANT: must return the total number of items to be read from the buffer. + The Pipeline execution will keep calling this function in a loop until 0 is returned. + """ + # + # Replace here with your code that populates the buffer and returns the number of elements to read + # + chunk_size = next_chunk.size + in_buffer[:chunk_size]["X"] = next_chunk[:]["X"] + in_buffer[:chunk_size]["Y"] = next_chunk[:]["Y"] + in_buffer[:chunk_size]["Z"] = next_chunk[:]["Z"] + + return chunk_size + + # Configure input array and handler during Pipeline initialization... + p = pdal.Pipeline(pipeline_json, arrays=[in_buffer], stream_handlers=[load_next_chunk]) + + # ...alternatively you can use the setter on an existing Pipeline + # p.inputs = [(in_buffer, load_next_chunk)] + +The following snippet provides a simple example of how to use a Numpy array as buffer to support writing through PDAL +with total control over the maximum amount of memory to use. + +.. raw:: html + +
+ Example: Streaming the read and write of a very large LAZ file with low memory footprint + +.. code-block:: python + + import numpy as np + import pdal + + in_chunk_size = 10_000_000 + in_pipeline = pdal.Reader.las(**{ + "filename": "in_test.laz" + }).pipeline() + + in_pipeline_it = in_pipeline.iterator(in_chunk_size).__iter__() + + out_chunk_size = 50_000_000 + out_file = "out_test.laz" + out_pipeline = pdal.Writer.las( + filename=out_file + ).pipeline() + + out_buffer = np.zeros(in_chunk_size, dtype=[("X", float), ("Y", float), ("Z", float)]) + + def load_next_chunk(): + try: + next_chunk = next(in_pipeline_it) + except StopIteration: + # Stops the streaming + return 0 + + chunk_size = next_chunk.size + out_buffer[:chunk_size]["X"] = next_chunk[:]["X"] + out_buffer[:chunk_size]["Y"] = next_chunk[:]["Y"] + out_buffer[:chunk_size]["Z"] = next_chunk[:]["Z"] + + print(f"Loaded next chunk -> {chunk_size}") + + return chunk_size + + out_pipeline.inputs = [(out_buffer, load_next_chunk)] + + out_pipeline.loglevel = 20 # INFO + count = out_pipeline.execute_streaming(out_chunk_size) + + print(f"\nWROTE - {count}") + +.. raw:: html + +
+ Executing Streamable Pipelines ................................................................................ Streamable pipelines (pipelines that consist exclusively of streamable PDAL diff --git a/src/pdal/PyArray.cpp b/src/pdal/PyArray.cpp index 0dc875d9..62b4875a 100644 --- a/src/pdal/PyArray.cpp +++ b/src/pdal/PyArray.cpp @@ -34,7 +34,6 @@ #include "PyArray.hpp" #include -#include namespace pdal { @@ -95,7 +94,8 @@ std::string pyObjectToString(PyObject *pname) #define PyDataType_NAMES(descr) ((descr)->names) #endif -Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) +Array::Array(PyArrayObject* array, std::shared_ptr stream_handler) + : m_array(array), m_rowMajor(true), m_stream_handler(std::move(stream_handler)) { Py_XINCREF(array); @@ -164,40 +164,77 @@ Array::~Array() Py_XDECREF(m_array); } - -ArrayIter& Array::iterator() +std::shared_ptr Array::iterator() { - ArrayIter *it = new ArrayIter(m_array); - m_iterators.emplace_back((it)); - return *it; + return std::make_shared(m_array, m_stream_handler); } - -ArrayIter::ArrayIter(PyArrayObject* np_array) +ArrayIter::ArrayIter(PyArrayObject* np_array, std::shared_ptr stream_handler) + : m_stream_handler(std::move(stream_handler)) { + // Create iterator m_iter = NpyIter_New(np_array, - NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, - NPY_KEEPORDER, NPY_NO_CASTING, NULL); + NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, + NPY_KEEPORDER, NPY_NO_CASTING, NULL); if (!m_iter) throw pdal_error("Unable to create numpy iterator."); + initIterator(); +} + +void ArrayIter::initIterator() +{ + // For a stream handler, first execute it to get the buffer populated and know the size of the data to iterate + int64_t stream_chunk_size = 0; + if (m_stream_handler) { + stream_chunk_size = (*m_stream_handler)(); + if (!stream_chunk_size) { + m_done = true; + return; + } + } + + // Initialize the iterator function char *itererr; m_iterNext = NpyIter_GetIterNext(m_iter, &itererr); if (!m_iterNext) { NpyIter_Deallocate(m_iter); - throw pdal_error(std::string("Unable to create numpy iterator: ") + - itererr); + m_iter = nullptr; + throw pdal_error(std::string("Unable to retrieve iteration function from numpy iterator: ") + itererr); } m_data = NpyIter_GetDataPtrArray(m_iter); - m_stride = NpyIter_GetInnerStrideArray(m_iter); - m_size = NpyIter_GetInnerLoopSizePtr(m_iter); + m_stride = *NpyIter_GetInnerStrideArray(m_iter); + m_size = *NpyIter_GetInnerLoopSizePtr(m_iter); + if (stream_chunk_size) { + // Ensure chunk size is valid and then limit iteration accordingly + if (0 < stream_chunk_size && stream_chunk_size <= m_size) { + m_size = stream_chunk_size; + } else { + throw pdal_error(std::string("Stream chunk size not in the range of array length: ") + + std::to_string(stream_chunk_size)); + } + } m_done = false; } +void ArrayIter::resetIterator() +{ + // Reset the iterator to the initial state + if (NpyIter_Reset(m_iter, NULL) != NPY_SUCCEED) { + NpyIter_Deallocate(m_iter); + m_iter = nullptr; + throw pdal_error("Unable to reset numpy iterator."); + } + + initIterator(); +} + ArrayIter::~ArrayIter() { - NpyIter_Deallocate(m_iter); + if (m_iter != nullptr) { + NpyIter_Deallocate(m_iter); + } } ArrayIter& ArrayIter::operator++() @@ -205,10 +242,15 @@ ArrayIter& ArrayIter::operator++() if (m_done) return *this; - if (--(*m_size)) - *m_data += *m_stride; - else if (!m_iterNext(m_iter)) - m_done = true; + if (--m_size) { + *m_data += m_stride; + } else if (!m_iterNext(m_iter)) { + if (m_stream_handler) { + resetIterator(); + } else { + m_done = true; + } + } return *this; } diff --git a/src/pdal/PyArray.hpp b/src/pdal/PyArray.hpp index d2fb9674..a702176f 100644 --- a/src/pdal/PyArray.hpp +++ b/src/pdal/PyArray.hpp @@ -58,6 +58,7 @@ namespace python class ArrayIter; +using ArrayStreamHandler = std::function; class PDAL_DLL Array { @@ -65,7 +66,7 @@ class PDAL_DLL Array using Shape = std::array; using Fields = std::vector; - Array(PyArrayObject* array); + Array(PyArrayObject* array, std::shared_ptr stream_handler = {}); ~Array(); Array(Array&& a) = default; @@ -77,14 +78,14 @@ class PDAL_DLL Array bool rowMajor() const { return m_rowMajor; }; Shape shape() const { return m_shape; } const Fields& fields() const { return m_fields; }; - ArrayIter& iterator(); + std::shared_ptr iterator(); private: PyArrayObject* m_array; Fields m_fields; bool m_rowMajor; Shape m_shape {}; - std::vector> m_iterators; + std::shared_ptr m_stream_handler; }; @@ -94,7 +95,7 @@ class PDAL_DLL ArrayIter ArrayIter(const ArrayIter&) = delete; ArrayIter() = delete; - ArrayIter(PyArrayObject*); + ArrayIter(PyArrayObject*, std::shared_ptr); ~ArrayIter(); ArrayIter& operator++(); @@ -102,12 +103,16 @@ class PDAL_DLL ArrayIter char* operator*() const { return *m_data; } private: - NpyIter *m_iter; + NpyIter *m_iter = nullptr; NpyIter_IterNextFunc *m_iterNext; char **m_data; - npy_intp *m_size; - npy_intp *m_stride; + npy_intp m_size; + npy_intp m_stride; bool m_done; + + std::shared_ptr m_stream_handler; + void initIterator(); + void resetIterator(); }; } // namespace python diff --git a/src/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp index b64ef3bb..ccac0692 100644 --- a/src/pdal/PyPipeline.cpp +++ b/src/pdal/PyPipeline.cpp @@ -245,14 +245,20 @@ void PipelineExecutor::addArrayReaders(std::vector> array for (auto f : array->fields()) r.pushField(f); - ArrayIter& iter = array->iterator(); - auto incrementer = [&iter](PointId id) -> char * + auto arrayIter = array->iterator(); + auto incrementer = [arrayIter, firstPoint = true](PointId id) mutable -> char * { - if (! iter) + ArrayIter& iter = *arrayIter; + if (!firstPoint && iter) { + ++iter; + } else { + firstPoint = false; + } + + if (!iter) return nullptr; char *c = *iter; - ++iter; return c; }; diff --git a/src/pdal/libpdalpython.cpp b/src/pdal/libpdalpython.cpp index e5fc353a..09118fbf 100644 --- a/src/pdal/libpdalpython.cpp +++ b/src/pdal/libpdalpython.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -189,11 +190,22 @@ namespace pdal { )); } - void setInputs(std::vector ndarrays) { + void setInputs(const std::vector& inputs) { _inputs.clear(); - for (const auto& ndarray: ndarrays) { - PyArrayObject* ndarray_ptr = (PyArrayObject*)ndarray.ptr(); - _inputs.push_back(std::make_shared(ndarray_ptr)); + for (const auto& input_obj: inputs) { + if (py::isinstance(input_obj)) { + // Backward compatibility for accepting list of numpy arrays + auto ndarray = input_obj.cast(); + _inputs.push_back(std::make_shared((PyArrayObject*)ndarray.ptr())); + } else { + // Now expected to be a list of pairs: (numpy array, stream handler) + auto input = input_obj.cast>(); + _inputs.push_back(std::make_shared( + (PyArrayObject*)input.first.ptr(), + input.second ? + std::make_shared(input.second) + : nullptr)); + } } delExecutor(); } diff --git a/src/pdal/pipeline.py b/src/pdal/pipeline.py index 37d98163..60a181c0 100644 --- a/src/pdal/pipeline.py +++ b/src/pdal/pipeline.py @@ -2,7 +2,7 @@ import json import logging -from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast +from typing import Any, Container, Dict, Iterator, List, Optional, Sequence, Union, cast, Callable import numpy as np import pathlib @@ -41,6 +41,7 @@ def __init__( loglevel: int = logging.ERROR, json: Optional[str] = None, dataframes: Sequence[DataFrame] = (), + stream_handlers: Sequence[Callable[[], int]] = (), ): if json: @@ -58,7 +59,14 @@ def __init__( stages = _parse_stages(spec) if isinstance(spec, str) else spec for stage in stages: self |= stage - self.inputs = arrays + + if stream_handlers: + if len(stream_handlers) != len(arrays): + raise RuntimeError("stream_handlers must match the number of specified input arrays / dataframes") + self.inputs = [(a, h) for a, h in zip(arrays, stream_handlers)] + else: + self.inputs = [(a, None) for a in arrays] + self.loglevel = loglevel def __getstate__(self): diff --git a/test/test_pipeline.py b/test/test_pipeline.py index fcec914a..35795ea3 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -3,6 +3,8 @@ import os import sys +from typing import List +from itertools import product import numpy as np import pytest @@ -728,3 +730,149 @@ def test_multiple_iterators(self, filename): np.testing.assert_array_equal(a1, a2) assert next(it1, None) is None assert next(it2, None) is None + + +def gen_chunk(count, random_seed = 12345): + rng = np.random.RandomState(count*random_seed) + # Generate dummy data + result = np.zeros(count, dtype=[("X", float), ("Y", float), ("Z", float)]) + result['X'][:] = rng.uniform(-2, -1, count) + result['Y'][:] = rng.uniform(1, 2, count) + result['Z'][:] = rng.uniform(3, 4, count) + return result + + +class TestPipelineInputStreams(): + + # Test cases + ONE_ARRAY_FULL = [[gen_chunk(1234)]] + MULTI_ARRAYS_FULL = [*ONE_ARRAY_FULL, [gen_chunk(4321)]] + + ONE_ARRAY_STREAMED = [[gen_chunk(10), gen_chunk(7), gen_chunk(3), gen_chunk(5), gen_chunk(1)]] + MULTI_ARRAYS_STREAMED = [*ONE_ARRAY_STREAMED, [gen_chunk(5), gen_chunk(2), gen_chunk(3), gen_chunk(1)]] + + MULTI_ARRAYS_MIXED = [ + *MULTI_ARRAYS_STREAMED, + *MULTI_ARRAYS_FULL + ] + + @pytest.mark.parametrize("in_arrays_chunks, use_setter", [ + (arrays_chunks, use_setter) for arrays_chunks, use_setter in product([ + ONE_ARRAY_FULL, MULTI_ARRAYS_FULL, + ONE_ARRAY_STREAMED, MULTI_ARRAYS_STREAMED, + MULTI_ARRAYS_MIXED + ], ['False', 'True']) + ]) + def test_pipeline_run(self, in_arrays_chunks, use_setter): + """ + Test case to validate possible usages: + - Combining "full" arrays and "streamed" ones + - Setting input arrays through the Pipeline constructor or the setter + """ + # Assuming stream mode for lists that contain more than one chunk. + # And that first chunk is the biggest of all, to simplify input buffer size creation. + in_arrays = [ + np.zeros(chunks[0].shape, chunks[0].dtype) if len(chunks) > 1 else chunks[0] + for chunks in in_arrays_chunks + ] + + def get_stream_handler(in_array, in_array_chunks): + in_array_chunks_it = iter(in_array_chunks) + def load_next_chunk(): + try: + next_chunk = next(in_array_chunks_it) + except StopIteration: + return 0 + + chunk_size = next_chunk.size + in_array[:chunk_size]["X"] = next_chunk[:]["X"] + in_array[:chunk_size]["Y"] = next_chunk[:]["Y"] + in_array[:chunk_size]["Z"] = next_chunk[:]["Z"] + + return chunk_size + + return load_next_chunk + + stream_handlers = [ + get_stream_handler(arr, chunks) if len(chunks) > 1 else None + for arr, chunks in zip(in_arrays, in_arrays_chunks) + ] + + expected_count = sum([sum([len(c) for c in chunks]) for chunks in in_arrays_chunks]) + + pipeline = """ + { + "pipeline": [{ + "type": "filters.stats" + }] + } + """ + if use_setter: + p = pdal.Pipeline(pipeline) + p.inputs = [(a, h) for a, h in zip(in_arrays, stream_handlers)] + else: + p = pdal.Pipeline(pipeline, arrays=in_arrays, stream_handlers=stream_handlers) + + count = p.execute() + out_arrays = p.arrays + assert count == expected_count + assert len(out_arrays) == len(in_arrays) + + for in_array_chunks, out_array in zip(in_arrays_chunks, out_arrays): + np.testing.assert_array_equal(out_array, np.concatenate(in_array_chunks)) + + @pytest.mark.parametrize("in_arrays, use_setter", [ + (arrays, use_setter) for arrays, use_setter in product([ + [c[0] for c in ONE_ARRAY_FULL], + [c[0] for c in MULTI_ARRAYS_FULL] + ], ['False', 'True']) + ]) + def test_pipeline_run_backward_compat(self, in_arrays, use_setter: bool): + expected_count = sum([len(a) for a in in_arrays]) + + pipeline = """ + { + "pipeline": [{ + "type": "filters.stats" + }] + } + """ + if use_setter: + p = pdal.Pipeline(pipeline) + p.inputs = in_arrays + else: + p = pdal.Pipeline(pipeline, arrays=in_arrays) + + count = p.execute() + out_arrays = p.arrays + assert count == expected_count + assert len(out_arrays) == len(in_arrays) + + for in_array, out_array in zip(in_arrays, out_arrays): + np.testing.assert_array_equal(out_array, in_array) + + @pytest.mark.parametrize("in_array, invalid_chunk_size", [ + (in_array, invalid_chunk_size) for in_array, invalid_chunk_size in product( + [gen_chunk(1234)], + [-1, 12345]) + ]) + def test_pipeline_fail_with_invalid_chunk_size(self, in_array, invalid_chunk_size): + """ + Ensure execution fails when using an invalid stream handler: + - One that returns a negative chunk size + - One that returns a chunk size bigger than the buffer capacity + """ + was_called = False + def invalid_stream_handler(): + nonlocal was_called + if was_called: + # avoid infinite loop + raise ValueError("Invalid handler should not have been called a second time") + was_called = True + return invalid_chunk_size + + p = pdal.Pipeline(arrays=[in_array], stream_handlers=[invalid_stream_handler]) + with pytest.raises(RuntimeError, + match=f"Stream chunk size not in the range of array length: {invalid_chunk_size}"): + p.execute() + From b569f2774ac0a868d637ac35a143fe8ebcffbfb2 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sun, 11 May 2025 21:32:05 -0500 Subject: [PATCH 39/59] Quickinfo scales (#189) * added additional metadata to quickinfo * use module call to run pytest --------- Co-authored-by: Norman Barker --- .github/workflows/build.yml | 2 +- src/pdal/PyPipeline.cpp | 6 ++++++ test/data/simple.laz | Bin 0 -> 18217 bytes test/test_pipeline.py | 9 +++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 test/data/simple.laz diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b460c7b9..fc69ec07 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,5 +74,5 @@ jobs: export PDAL_DRIVER_PATH=$PDAL_PLUGIN_PATH:$PDAL_DRIVER_PATH python -m pdal pdal --drivers --debug - py.test -v test/ + python -m pytest -v test/ diff --git a/src/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp index ccac0692..7f295273 100644 --- a/src/pdal/PyPipeline.cpp +++ b/src/pdal/PyPipeline.cpp @@ -185,6 +185,12 @@ MetadataNode computePreview(Stage* stage) } if (dims.size()) summary.add("dimensions", dims); + + if (!qi.m_metadata.empty() && qi.m_metadata.valid()) + { + summary.add(qi.m_metadata.clone("metadata")); + } + pdal::Utils::toJSON(summary, strm); return summary; diff --git a/test/data/simple.laz b/test/data/simple.laz new file mode 100644 index 0000000000000000000000000000000000000000..6f774c5b715aa4fa8c60b96b3d3f86f6cc9cfc56 GIT binary patch literal 18217 zcmZs?Q*bU!&^7ub*|BZgwr$(CZQHhOW5;%OY}>Yz^VX@a&iU)Bf7SFfrfa6xT+Ky~ zjG(d@;Qs<3U>QMW7khgfX95aiN&+KK0w+TUOA{MIXJ>l?aa$t^z<(ywe?mavX6oc@ zX>Uis#Kg$R#6kIgZgvQe1p)y68-QRU0OcP5AQuDxaQe?h{ZByuC(|3HPJG`hg#Ul* z|5*Vv7%kO3>WQ%nbaDQgF4{ffq_SNPrLOKg-fn zpsVV?GQBJv2u$sa?M+Ng0Dx{a7Qlau{wHj2M)3c0{r^Y+|933Npa07J4+=v7!0-P8 z2mlO#0ssbp10epJA^!7#0odaIZ#v)%@DHRu7#=jQ0sx2yNCLfvoNfcX5)^x$596wh>p7&EUkk&>7QW)Y-ZTH;@m8hFZJGm0|cNi&zL8?Oommf>s! z_GXFf#9)-Wl;sO3R;Q9sG@K$VK_^XKJ}74X5guHaTBiR zGU78k$=7Am9WbZRL&21l!QWxWy%#gOxFR(4c{NJ;sh92vh;wszDl?Zeh)O`5V;Rv2 zI(q}Q3+r|m0%`mdYi{=RTf7U-1-b{4mkC0ty9Lpu>N`Ua7gX@S$RB2gufAhkGFW0r zD(l?S=kOe8DNmnvv?%n;XUNq@PACozy~@agWNlNuk3JC0hP1ywcxLGmkoXDQ65fIl zix3feIXhkI$-vg96oFh>_w*2*e-(eu2Q5zg2yB42u3mTBCLM$8nDx#4)s!Djl0V0gd}J^Q1#FCUdfojXsa7 zU4p!i@6fqx$VdXr^ukJqq}GpQK7s^W53DSr@;;W;d$HFUZq(BE_&-8bEH{C|4#Xm3 zMayOzjwnFEMVc4$_9U$*F3?8~plqT{_?gapl`00jj?>t619yFav~T|3E$YS90sf6C(jT8! zz%6hbbar8_J!?o@vH~AEAs*Gb z(N?UFu@U~X6~tZzvSzbQ^6UNTHsU_&t@~^&5=VK*{8l_rCy6h+6wp~x%R&nvLe$H@ zC}vtbsi{M6Mm|<-Tc|Rla;Xpw+s}g+Uz;&G{L)rqiE{C&mnk;gK-7{!SW;B)ABYE{ zBVg|T%cL}Ss7P*@5M~^FL!~&rscrs8lk_93zXsED2sw!`76wNRv7i*qYdof6`iFI4 zYeqP{*Uq%A|LWHDkJ1hU_Mu>WccH*YCS~OyEczCWUc;f4(FT}uqSPZl*5`0_vGpX? zUkDNDhz^%eX{;2ruM6@gb^a+^^ulN+c_YB<4 z7%flPzcZ!e7>KOB4*hVLa<9WBF_Cq`sbGsCj;Tag)n$~2FjLxPB0w>8Sr)GasG7R# z8RbQTNKcmAH$b|rFMQAyD{D1-qfhk;x<;)lo6J}7F`hJAm)%MF^N z?vwm^6)vI05PiL=(A)=I-p#dCY6bmszuKJf>#qq3Hww`^K3 zTY_L8! z>-myQa*t+rhW|R1FMN|>d*xoPk_I5-C`>tQ5+nb?|Kczw9-oGkf8Df1ahSL$dl5m$ivnmyF zf93`K16C~IaCc)FuX>a*tgb^#tXdA5z=S`67`z$zlw?3ggzOh{GO~adm zKx|UY0m9}K^EOL3ow&#v*l@O)qwG0!h2I08O&qm`Rg0Gg#|f_YZ9OC29?E-rE!j&O zO^cj=-m_SBh2pnGm_$0fyvgvsiGj@$LZ6~tlY|BvzWajx5Kz$dnlfzFtoD6m$%}aq z3}dN7+P@s5J!M!oURhCWh_gsqxu}ELiw-dZQ2^CL=2yArsl)3q9$MrRYSbxgP<>m> z6wqEIyIxYOD$sV8nbi_Z5Vxre#9^s#emJwIaIyWUX^4iXxIo@vS;})#J8lq2tn*Q@ zuhJ&awyO#L@@9t5WO6-d3Gcnd+rp2>+DaF3MXV=!k>A0WBw2(VQ5u?n$)-p_JZ3gf zW=7qV2j8Q0o(Yd>tXKI&b&;nFI4i)nUZ7^z^18iT$U(3FgF&?4XY^`ZKFNNG@CIkG z`>((eQMS)YE+b)|#)0N;Ku%#T2(l9+>*y~KPVRlDrUVyNxK+@ZdjMSNQ}k}GAvy>4M05=1UV z+_$SO>Kynb9pmjuq=U79F{gf4+n32hCWMI&;XHNto72?LRR?*k)BLr%ae57d{-y8W zpn|_<=$_QMz$5in_#x+-Pu`(|T8LA;a74j+ z^?u>|;H?Se<1NGF5!Q$g852x02J_sNL&%4y-zsSyZlt|ghj?6RI4xcm*qTo!? zU{f4^N)G@7B>9wneW|ZbP{M`S8Up)jk9H66}EJ00tvSkZVJhQnkMGlp92`4SX$p+t z!h!O+fc2b62n?BZ!27dZ)BNbcYih)sESFqp$4F&_$K0qUY-X(LM!nBI2PL@!4Slzf z4lr6dZloO{@*r&z^SLS2DX-uKmEUoViOT1X7mqL-@NO zcHGMGrAQzbxdSdC8DQKAbkZUW2UF|gul>)QrZm|rcs3LhvTnLugP&cycpOJE=D`hrF?7fqmXu{*K&AyH1}&k7MyJI- zM*EpLJv<4A*TDizZMeI*w+M1pM(XH?fh+845WHa|X%>3=#2AoF(lMR`_|TV2$|{QW zwG{y{5i0XFeJK#6zTH=P?=wDmYZ$M#->3DQgRa%lYu?HJ!nxe-4- z43V3MpbqjYz1M~JjwmvLb~$T$6(>dOga!^ALz;#R2HkQ61(R2Q8VT9Hg7u)I&p7P% zlm|LlRH29C{XXcYPq@p6MWfg@Cu%%*Lf(3#xopY@fFAc{v|lB)>Rf>GhdD$|;DKxa zN?I>Ah$qYIffP02hyjN*o8|8=SQGYW_;Lxcr)#eYTO*OV2xNtcoC_`Yp}%dVD@Lcj zD45ru2gKHIL#xNCQ{}jgDS5F(`ew7E-fK*5X6zZRS3~SQ(yMegCM=OqvM^e0{b#5E z{Uy~DI*|{{$V-=BJhC8NqKqUVowLLW5@!_yui10(l^OFe?pSUnTHv#)VN)A5(7gd7%x;g@yn zS`3adBfP`LW@Is%l#|!d_&ykek?HYxi?~#YFeK-xU-%FMn==;9%t;YwfT`~p7+O7U zP((Aijoxo5%}Ud55sXvF;4FpjX?q4uTJ@*PJx+f~+uiVgF@V#v*$vZI9Rchz(yGK0t?-$FK( zNxqq*CPJ?_DbNtujJ82&gGRJ=SvQD_?O(fGa?&koiCas;PiY+i;(m{#Wa*c{rs|NBWyDAk0s2#Q622l!0n?t=qXJt z;QjbIj;;bTt=i**#C*i2AURg5#1G583_++ht78SQ?vu#nBf0i+x{V8ROI(hhdc>N- zkE+6uAp*JEh@p=P;iWBkRjc@DCfuY%G+PUCcMF>HNO-I5SX=+7>AiXg|JESDZF$gm zjD%~S;}%e*B+tLv3ztu~&Y9M+rkYUWxQs^mW%RLjO);I=i5Gi)PHV2R2k-zJ#O5`0 zb!v^ZA0m@eoN;iNxOB0Q!4ieh$p8hi>iCR6PkGDgU)vrM*J75nI`xD-M>J4y04chU zubLKnKxM*6*@E&H7)xa8G!vYtP%aMiAPYEsN@JD{MzUvS?JiyD>iAWyoiu`~BN0&Hdm)j>*aUQyGg21G=O}qf>?My7aLRXSCLr+lIS_8bfax zDu+{95G;B?AcAvFFGNigm#{tE=peefVBUeMXkpl?*rh~sU$i>?ui_|H7XQ2IBYv27 zm1whr+PBq4oT<&M_4Afqz*mPu^BE%OAzHiy z`Q~THCqto6j~zZDVdI-l*c!~(CQ?%>luVj-_uWRH-+>}om0a|{z~Hxf0H=nIO(*NF zwLL31cys0o#ZMwfq?sFC{0zw}sJ$5L^T0dK43+s85i9e!IvXcUjaR#7+_}CcI0Z;gnkn1vw4FKsXY6jD3xYTvTd77m{FCp2a~ZXwoQ%boZtHwT zWL&xDNk@|ns@?$GAJ=Nh;ZNIj3S~?!a>u{e8}(nojEgtimIqu=2rnCNBWBLKo7qT; zr0tnk<;FZWBRIZ0JFj@7X5cP#^?JVn$|%UuGhwO9#a6h2UX%S1qVaD7x)tR4>*AadjaH2)Yb9(i|eK62O61^_u|N%*?3@vPyYC zw|?akGDfy;I-ekgo_9avz@C6&H>$$A(Dv2XTsH9$G4aWN|2eNYCvK*FQKx=kgx#Xp zzhe%^Pc{4+G;f1pr;3F(A^p=Z;~6)N7B53guXB@p&X3G-_r}Ee%TRNWq2H4~pr9;^ z7jTSf%1`nFBqcDQVxVvt%HFG|j=U@+rBJkrNM?`OSCQL>uW2cKJoi}eXl`?*{bAWM zQdjA@FQGS$L~W!}KwqfsAxRuMu;E$WURr*h#FzkKo>~Qd5@C7adw~ePc^686;PN=! zNKGe(^3#1>3S}8!(}2wb@=pv%@_6Xfh2b6p-LJSn!M0KhEl)$mD**U4{0}=rw@Y-b z?*zxCDkgmn4-ql6UpzyQtwEgb-8Z-$rk&F31NN)Jn(m;}N#^TqnPG)*^phl17!0l9b*Jw3k0b;3XNyxV>}FdlCH~^7EG%QcleuBRX44uhkgrh z6^2C#h2F_i96{6f@Y_FE5sF0>ofx%@J9yA4ddCskIC_lxK9{{H8ESzg!2OyuYxLw6 zCx1qu%xl+tNCpbD2J{aK$mf@Z)?~SzJn(3mQ-517l^IIqdJpF}NGd%oC|*%9e2+j% zh8s#FG0>F`pUflUKWmJLmuVm?lgnaR4dW8HA=!*-?Knkq5?qJ~6N0x#mf zO&}~0iwAG3GDP)j491^*x{2EvC6E4h0hR(`XBJjFwmmi~w5V;C#II>~&De)_MnW_f z^rP~x;3V36g0}eMDC(u;{OzpCL1M}t%Jn}$a3m^;Xhx(nvEK=?)NsjXTokCs4 zLp7TGDpfPN%=Y-1^kBX6kZ^1tN+H|$u=jqdlBiVjoKVU#D8wTxHbCc+%>Vwd4|T!X z^#qzL+1;0eDG)I+$k@EEJXtlfl%P@qlps`xWz5(E&{GUcZ3+UFy9eIs#Ts@PeuHff zNz}!g^M$gn+)u=V5|tjU%9-N#VMCMNF2W51ptN2HhWZ~1uP|ZJxXeacw3^Z)8SZ}-U9R3pRN}z)4RPM{MfRjQIzoz*QY6v|xVx;i3 zx1-qbMvE8&v2nHUiz@%A`r9#eS65$C{BUFC;5f(-NKz!o zdw*{2AA^2z0h9%Xc}=91?qJV&oOo8VvE|a4r=D84hCKBAk>IUpf>(uiheWOSmZxBH z>SX$xB!hWiuZa>p7U!@w&UKh4Z=%bh)?b+rosqjPI1M6Kn5w~OdgI+~MSd*QnS&8< z>0~2@&9i2}m7J;RgCCr78kuJp*-_6(P5EKe`E;JKnZ(-RDI0fe3B*cQfM}Wr2FEP3 za-gj$!#4?37*nVmW*80nmA2f|ziU@rM>0tL5}jIlESO7~e26{Lh!b$z$<#8H-|kq# z<=R-$vAjN7J9`dso-6;au~BA2m1uSRUrj=n7*IguE_whxkpMSVI<$x|ej8kM zr(-Y+ZP*XY`-0D2b5Nh~ceWfEi05ZH)p9Fw8939X8 zR~D}A%Y6TKe6TZWYs;LdeHDW6^iKfd5lM;CYp>s8|{az_OtKL@L`3LlV zJc#bdbbyt?ZL;W&2fT)|W#IWaRWVO(sj2;jgjeqC;#mOc0+mT zE=9w)v9(n<{0iA$8{MK)%0D;QyPdyOqsDg!1s#!lSG2(Dl3$)flV$Z0<6gNVYF$@; zw;pi5!!mBx!1!IbRH^v5uE%W1;p%N@lw)e}@FDTO`}r+hI83=@rcdRaqvK*$tpZ~y zD{*+fnQ?XKQmSPDwK%4{`FA+6V9dla~zqEUxbIv}uB zq#R6}k?il-?P~!YvA;Ca2HgG}$kYZbM}3bG!U2PCt`)=+!~?Py8f+r>NriiSPS+jt%6KQ~znCBtJ57-S8$ zFDk-~f(z?Pc#IS|ef1QD6_w&(56 z8y&(8Gx4!Hmcm)>Ko^Y{xG|)v>FuY>)VB_NqkAS`KXI}}F5`ZTL-qrLVfg9jBkkE6 z4U7@@>fO<&-2(Iw9)>#kn$+mF5rm3_;9PktK5sROC3hm;qd-_#c1hJ$5AJyhBgh-} z?5WTB*~vN|Jl&N%B=!bLrjd=J#Im#jyGQ4sytX6mbdKs!f>tjf2)eW**WLnhg5$&Q zHx!S|tFB+1osKBO;^@SEblEQ~j95?7Ma~0|n>yFWpux^lkX1rddl#664IWwZkw8}# zIpUlIwX4&s^3vDvtVzmxbvohMd*{{#( za5joh^rxhYk(3h0d$i%tj7E%cR7(Nt3SY{`%%J?v){NEvf+cvLurl<(IF6s>ND z5bZloU|jNX6MshptXCw5{2z71rr{#Rv8%qenAr***(tLTq+CN5g-Tx`0IYlg4-Z-^{QbT+qR*pQm zRRZoB3kSe{&B zls}6J^bkY@i{8YeeRJ5gK-%WQPauS|1d0hQ=P+KI)J21bdTh4w*+LIH$eR9+J+5I! z+Rqx)5&0p6-}Kz8hD)6g6t1lt;mGMV9in94F`(MyVR6udz57hRA%kDv1DmNAa;bBl zt3c?o-W4I0^E*Hk8$FH$wj^aL2XL+#clT#-R*r(v+wnww1pg{T8t&6}9iwFYOxsV# z2~~OdP4W0b8yMQ(&pJ)y$5cC;x`GD~RF)H+>%5Wl*=%NcVCoJ(cHa%zjFZDuvt-V{ z;FAV@CaP(IX-8=qP@2?I3#=T6*n9R(Hy&Mq-S<&Ctr?Bj9G1YBi=Y{nN9R5{ZVP}2hU z!kHZSQ{Q_AAt1Odn-P9{h1QJaTjhK;J*-l1;zd4!;*;y7x#CaH-G+I(4L>=v55@3q zvC!QZSicW+ud>|;Gk%ZSqMWv?`6WLaJ8K1h=^M1B2^5#63ON|+xlpd~t#hPPU_nPRkXM^VPmWn!j1eu`r7VJi}FvLTn7&4Wv<65>JMADWsWo#&kgNR3buIPhu`?syAWLB+%j-VhM~I=vfrhV}P$BE)UUzG)YY067*$ zh}j?1qS{hrV;8EVq5{_BRpcA=fK-T}OA_Oszt~0_$aZ1;b1mAlfWQEnArK&5S!++6!J_7qVkEcLy~%QktR&S!GOV=o^7+De+(pV&pFyhjoV0zr zOypqE7L~M0NwN?8$yqC2+M$Axj+PLbY*?$@2`5NDsWCZf2F{5UT;HJnDx6$GO*ul- z;;zCC8@;@>f7o+TiigP}J(WkG<=;JrD!Gl+hJf0b=?f$EtXxka(Hq5(V0#(VUB>praOYVG2@zy$B+@x zdF_UV5RW3BS4Fm-UT{PBe|`l7|A5rF;xl4zmquK<1o@$CE65p4k(xApqG~eN{rqTi zt^_T?--5`rfo0;GRg($(7^IqLn%3KAp9o+%H+6-%3-w|O1D4-nQ-a)rUBJpk)EA5P z(OgUt%dHW2?=ubj+8@EY@eV6*m&d?Ni8oRecZr~j!;Yc{^BxGH?tZ?LcAX=?euZ4H zC|=QLnm{vgd18TrvP=V~$!ZIq#&w(Dn${V4`bp5StV|^&?4r7mm$DIsacIp%T7+=m7?yL3B}`>y===I{O`w}5yL z8vmWnKQ1FK!u-7Oi$SX%ovA-X+Vx?a>Li*Z{4=#s+W(!9rVhnks6D1cKhu8NQBMT{ z#bU2zZV28@XtN~8jId!HK$GZ~GV_n;I}?-w9BLO?Xg&v_Qiz&h+k!+^Fm(Hq$WfB; znyB_BoTzP*)V>8ixMsl1bL7p$XRO9-W#^gXJx;RTnF#A{mk&kvv|FlrrQ)PifhKKK zCqH2`>B&{5&tFwtk1|L%c!;j)R+`e}?s2L--LOzU3{E@-f0Mq@yh=F!=_=C#3^dh5oAI+O231PJ!$JtVB>D!g#(LH&>s~|Qx~x}aj136^M<(X zS+}FpsXs#7%!0zZ0flqaFaz z8YO-f`ceCXH7s|%0$|jq(eE9hctR_cX>Sa>NQQRUgl1(??y@Xcw3AWwakW*%U|4t2 z>Nw!_Z$1N*MvuS+G1JE#A)~oxmGwRful_r_pMu*0pRSseXy3OA%<_V?fBfG`_j*)u zW&84&L4N}{Md8Bt`Ly4LT7BZH0iBU!qFx)9Is0!A!gNAR1NfpLmLR{_(D2l=Nh8%3 zLq@%_gRW+)WCRgC_%8H(LK8&1KQ#BQ_$5o$S(h?dk&{FmRf)_;OD}T4CkH8|cSffX zA%U4>r?f;exB@>c;zw) zCYgom(#Wk~RBCq6(P_kvLl8$-FkQTiSnqBu^Y-Rl;G-K@OEtd^}YHmY)}8kIKbvQKH_IT0F`s9;hD zNOE(Fw)^Iw3b3YlUW;ViF(6smRJdQ=wj?IU?m^Y_32YTRGTVDdKbB_8Np;D=A;wlgLw5(|o8YWzNf+oM?sK6T%UfPQ@0N;*XS(EyAx z^eZV-2(r3#*%TFARyQPw9}$$?h#S#sxhehI`c8R8e5oyw0}%1q1Ery(8w zjV#ua^P6#c1*z%9BoM5Gzg(jR929P+&&QZFPAZ2U2zWNCB|drD}p|IU7aCEbPxD zv9&sMVE)V~gm10fgD9jC|yyE|2%lgUA4C=)lqHi-p6H5OYl?Abe8BuqxvrCc>HKnn7o%-?X zD@ACb5>gIj*rFBp58`gE$$VY=9X6)p^98RoVpcJmxo4^Va4s|v>HUgxwTf4dq6@h~ z$5d{ip#-1jsjJR+UPi>QFscy0@5#m>m_$)u9$4rPM5VTKC=CT|M8))E>J zGEx*WioER6?MO`CSKtlVZ?)nwStFS#R)mt@$QW-4`-4xZ=U(7s!`F1I8^0?U$YusnBGS1gCy1DXB$rP4qU%BITYb`&$wS za`q3F;BD@#E8`0O{G~#n%bBQigO4g=$<3=Ro?|r>dHp42W$Bl2&^FoEG%-clKZ32P z9+WNuK{i^Al>UUmaiU(<@>!)6&YiAn=GiXzhSaE9;%Vp%bcp5nn;1=;kD5VPM}mF@ zhTMRRkL+;*`p2K9K$IkQCmm;B2%OE*q0xEt&pn4igfK|4x?o>sWg9Xw{1o@IbmTw@ z*Jh>x3-UT9S9~Wz(Mah8@wlnO52%P)t6#A7%Y!(Cmdc+p2#kXi9G38|j#GbRpNnwc zzW^-+Pfm6eEYU;Xl4ye8P_j3L#Ut3dxM-l|r0ssN$3E5=OJS8z8x|{+Bvs7UN;WR+!(WRBlU)l!y=bAB~~}> zCc#@bCLui{Q0hgWz+>TUZ~_(wTX?5Pcz^T~KQf?#;V<|n))Y4}9f7{_94PA}n8{O| zr$SKA5#U=7uHM*ghMIh_ste-W@nZ6Wdd{x{6F)Ao%|(l*hCb{B1>-;9kWO;Br2&6> z5XngBcd#l+rga1gyicbuGuE<2eZGp;Yp%)ieoIG=R(+aOYS%@Jx&PHAte%9b3=M*D z$<_Ox(~GD$Za(V2H}#xgup9+bAA!P*9#3}o24+k)7%4f<55$s{c)Cg`HbbP*QJ77y z)g;*hO6Y-*@tKO=NiqDL4X`tb=cQUk(Teh++)B7YFpcT3csg}M2dZ4<8Yq{E8Ae^T>TehRJ4|aj9gmgrQ4MaEN zxd~~YO<%9`#k47D1+r+Lz8t)IyqDzQghByoH9v5;iz*1H4xGmFlg zi$OdmmN3-g*%$Y|!bC-E7(_E*v(34eXGzpu=$*r$^*bu?AYfwJRK0bbJM1^FS56Cf z;eo0Rbsq6R3$G^2n-JLWIlZt?A{t42g%->om=+x_2BqDyxlu?N{yc_(EUhgFYE;Z+ zPogZmAVUi+gw^h5n-EBJ<;G6lYO{Ke^R=kls@U^ z>&A6^1hCK?e zP~dTWNL8ujae^OOq)&Z+5JBNSGa}diHEYeT$041(tzH6`v0XA{3qeUh@pdvkWZ=|E zHL-72p{^bk5BAE%OWZ7_;B<3G6J6(Pz5aHT|Fbk28S$$vuOR*Q z-34!$+B`}0uIp}BStzmrA+q{RX_*6St6xPxCBkhv(Fj1BMIK#Ipo+S%{ui^Xdv`CxsXI2w zPf`&Ld+=_lg^sF2uGpEnw$({1lLH~-WmJY|RjL*rl?8%zxzEc1+>`Q!3;*2+FQZDu zmIj(xdms^ojn=Lba-4EarewUv)X~lzuLv}tX<0f$zh7^Ig93p@82@jsxxl0{E=|36Yx|^>`X0MjJHPsX*pF9M( zyg}FS0@PnNA?v?agZ;*#6)Q|f)z!f{K~HrvZN{{qrM3}qivT2dl5K4Yt-#H5aA864JlMzf=nBr}8fB6;;-oUD~u6B6W! zqzH+24{Z=TzJV6KR3jRt7J<&mtZsW_yUmLLZ9Qh86F;WQi!^et6|n>De;&vUQ-3u#xpMo{X3g^R`8dO zc&>{Dz}?0|?_!6Gd2QyeD+*mq>tiQj?j(!2@Pnl0+w?j)a^_qeIT*GiQfWpA?V?-; zfpk3qReDKX29C!#$5eH?b=LE|Z7Nm@&G+;wysA;@5nm*#2T9T*CT6Qks|HRuS(BBxQIJ|ok?>6jInU8woTNisz47Je zP{RY5y_1n_QwM*VTG1SP81x=(iWq4K_%@`Meg3YZPr&K3w?BRSY!iZ*%2>z-MQ;a+F+NwQMLl$I)NW!v&3OR|N zqkJ;z0|q1MmQo2`U{61PRLAza0yXGkWQHAKfg6VQ+%Y0TSvYqD`Y$-C^U(}>x>9Gk z6lMSf%KT}_?rT~vv8KMG9<{!u5FGzgiU%;z>kHk?_jlgP@tx<3)hXp6;%eIBI_hv% zBc2GxkVz4WyPJKw)B-2^7Pb!~+}ytdt*O(zyY$FKle0op<9g*P6laX6Csodw{`)#} zaj)n2qiPR%LT4(vFX^8`rrw4S1(Jn#QPCIZn(hpIV>dS+IR1niSa0(ejwfA@X2q8ZpfS9`^}=LaX^iiO>1{92G-YC%?zz)vv{hAcS&5 zQ{rBJH;YD2Cb)EApD%Y2tNa@v^A!*X5rA!+$m{S{|>l|+zVVX4|&n?o^WDshg) zx5aCjhFcU6eWiwn^YpEi&g*gd=YSw+(6lM0fF+N;%e6}3>*60uNtxTCz=4Z~v|V=v zSZ4WR(N^YkRasMXl#_>{t764?SYVr*CAj=3A8R+Gxf~!?tR#B9o}fPk2?kz13y8`R z_FjJst%T^oqZ{As(o;Tj6(-*{1i}09$@5+G9+$zNv7})>gUDoVog4W?4v6A-yGU zesCaqA)3mb1RPwI>Y)y|O=JNwH;1>AQbbHg+?X0Ql*)#n{PEtyf|csP%X(y@VEdY3 z{a_<8igDDtuESmiwLi{%NN0r9lDLj}WG$FQOdeFU(0N(H5TDB}^YhHRmbcKfViZ5~ z+=Ej>|4coFojDZ&v~Ne`&=K*!%Y)&=>&FTG4kqIl6rQrCRp*TCiPY)hM>pocf-@XU zzv#gS=7>fzZ>cMj-JSEBVJD_9T?NsO;g|=upK`)5EYu|=dy*%EvR03hidTz!sSENe zze>S5!@wf4W}>}|gaAJqM6ulYnD}i>URKzZ>{-7y$Ja9&sO7SS6%|rR=icGB&LEf| zJU-nYx52g;f7N-F;@NDY5sqQ8PQ)_WaGyfvat{zQujfRGx zDI=b*zSc7N-|F7w?Yy{lwzeA(g}_q7>!W8QH#fgtfLQ^P4VL1BdAOl+cZqIvp8Ldx z0X~}Kd7kSJ*64LGzd__ywx%3^2ScbQ>tHr$;3$^lU-?E_U;R2*)`&oC{R&L}u!bMe zO%kYj$QaT)Ra*!`qo=4&9RDDYxmh@}H46OW@e{TT#3i$(Eqc+#P@wq!k#^7BvkyYQ zCGs~9w_ZuBkD(^GPL&R;($FYKB+|n1@YLjsbA==%+4xQljMJyr@i)f)P1;*ep+V#LqsK7O%DZe#qWJk($| z;Mzb->K#hpHa$z|=@KmxTFYr;h`KTZ%Y0r`9Z%Ju_|n@;s9d}#oZDx+bQ;YZ zlzZSX@CcuU#%8^dw!KI+=QIACG<%hGRiS|j3zEJe=QM())d()C0r5T_iniok-W7(^ zaNG0URdbkgDO@8bUwKX2EO$#ua-G9rB$U#H_+;smQG&cKIgX0&B?%qh1N*PNkiVUW zvlQb%o^ANN;`}!*a2WCqQ&lOD~7V8vzvRoI~q0)SA z2wON_0;Pep$GRvfD{V)z4$-5GfBXOY4fxG~C8XMK$ly=@00*%qyOgw&AEF&eVXZ#6 z5knel*~Ye7GWb^AT8PffS*E8zOy=^3HePOR$?^*M-v5gk$wrpz3E^ z(?#oUOy6exFF1UJcDwET{pRrl>h*jfUqy-VWeF$uEUvBm*|MyS)#pwWK6aUo_Zih_^n9}qlr-Zxb&3|MIG zj9jocDK^J@@XPV4f$G21B-H208-BGzQ2scI60kUe34An!u;t^8$tNW%S-!osN_P@t zbW=hLGBCr5OJ1g;@5J)1H`uODoyS;+UBs$ng3Xekn;7IY@|7py>AhG>>5~(m1 z0}ZoWkRKgTc>C9YnD)BF3N#FiGw*hy_|EaTraM&u< z5@`UzDIn}OqJWg{-eo|ClRPK_2b`>Hu&0$qiLy@ZlhWb_gwc(Wx3{&cFIm=pi{-CP zw9*VKb!fx`>6OMvYls|oW1(pPhZa$DiZLx_u(6O>r|JdzMFBoKFdb^?PEz1IiYAa^ zT(4^6`__IJNyKP9ynO^doZ|u(1KjGVt)8+~cVIdB17EppEHsF-cHk^RtUD{v$^NlaWJhR=nJYE& z!pqhPSm^dl!iiKvEhBy3S@0NoZ*xnDG+=Nti6#fU_ zPcTpVi*mF%?cqp&0_I4$H`j;Od0NtpayjI4LjnDY#McWo77u|mdp*(a-*0)cT7tc` zsXS8#`!YPw#4ftqcCh6@fAvL=FX>MN)R9H5*tZUwzy}-qVb~%@wWCINxLMZD%FaV^ zL{}{x;BRyLk+|tnl=7m&1|0;+hk#%_{U;GIgP7X^BH^=5(A)*HKIR-U! z-i{jTC`kwjv%`S{5EcjL%6N4_G@tsLpe9AW%dX-TzKk!x=5aL)mbMlpOnOE?^Z|CU z&i@2mj2bJKX)ZXd;-!2#fB2_3Sn_l`Y6kBXBd1H*7GUV)DT)?S$i^^O*$sn@6iQT_VmLz+pU7Wp22`5)_HTY}YIPp2ao%H_+ zmIi70x2AApvvFFAb=Pw;`qE{H3fDcH)vyTWb&be}X5UAYvdYPWB=jbpLx!-gz|tRH zJ)O#y$Ozs3P?YH{3QJE-h|PsN5C{1H)EZM<@=SO2R2e~7bc&yLb381NAHMdCw!*>< z*vvOzXACoz!=es^B|;tLPm_(!27U|_J3iY|nEUVgOf3&Pz!tL=R@Z~^$9A4I1$q9p zVC&uiI=yYRGFyh12tDT|wvv%pB*QW-eqDbyGJ4Qb?ndGJDc)wQX+g8E%fu#2Kc;@QPbYWb?Mn0*Fhml0gM{6{ z9qMnz(m2Z~#D-Nk4KLa37M8-(>x$kZvEfEvAwxfR5$^CA;Ky1xtzpIaQx(;IkS9VC z?}&vVmLfJ>*gTlJz{VTH+Z4;^U%JMoPU4cUl_z3PUm)zO(`uze=TFCvYAShivM{Aa zw#T~%rUl5E&f>NYUu!ujR(FLQxL&?pU!dgFw!`+cR!j0xFB%9#^xrA!tP zKs3pb@}J;YSuE$UpVwzZvJUN%zt)1?0mShVpe=cN!|6T*8JhsM2~q;EPS#W_a2z9j zCVkovP-;HYS-~hnc2W{Zo=`iP>AXw_gJl^k2_29-;V%XY>ay-!L05>Adz@x+1rM>Hh_;}P+q2Cx1%>3 zSpG2)*)K6!)$tvx@*=F(7tm$-qIcuOOJV_|}N1kVFICYd@8=h?=DBthFZ)qdkgYm%x|00gZ2Ls%E-qKk_-uco!Sj!~zUz zq|{o99cEvima})YB8bw}R%(}jLl^CQAd@lo^9;-pxv1tJjCfg7g7oB9SP;nhohmhj zXzPJ$)L_HbLy{b_FU*)vV` z{Md%?6l&Q5=g(p^S43&Kb@9{A%oWO%$|-Ocj|&B6r-2l#so@sTVp9L9OK-Nhks5Qw dp}2}Zy_n5CqyPW_000000RR91c$NhK003mY1Csy% literal 0 HcmV?d00001 diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 35795ea3..317c5b69 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -396,6 +396,15 @@ def test_quickinfo(self): assert 'readers.las' in info.keys() assert info['readers.las']['num_points'] == 1065 + def test_quickinfo_offsets_scales(self): + r = pdal.Reader(os.path.join(DATADIRECTORY,"simple.laz")) + p = r.pipeline() + info = p.quickinfo + assert 'readers.las' in info.keys() + assert 'offset_x' in info['readers.las']['metadata'].keys() + assert 'scale_x' in info['readers.las']['metadata'].keys() + assert info['readers.las']['num_points'] == 1065 + def test_jsonkwarg(self): pipeline = pdal.Reader(os.path.join(DATADIRECTORY,"autzen-utm.las")).pipeline().toJSON() r = pdal.Pipeline(json=pipeline) From 4f36e7e658cc2fc8b355d22feb7f9c6b51889508 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 21:02:49 -0400 Subject: [PATCH 40/59] bump to 3.5.0 for release --- src/pdal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index a312f568..f311ba7e 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.4.5' +__version__ = '3.5.0' from . import libpdalpython from .drivers import inject_pdal_drivers From 55727d33238bae1cf6f8de62e7f583f5d0f78b99 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 21:05:04 -0400 Subject: [PATCH 41/59] use PDAL_EXPORT instead of PDAL_DLL (#192) * use PDAL_EXPORT instead of PDAL_DLL and include pdal/util/pdal_util_export.hpp to get it * consolidate export stuff --- src/pdal/PyArray.hpp | 5 +++-- src/pdal/PyPipeline.hpp | 3 ++- src/pdal/export.hpp | 44 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 src/pdal/export.hpp diff --git a/src/pdal/PyArray.hpp b/src/pdal/PyArray.hpp index a702176f..b2aca844 100644 --- a/src/pdal/PyArray.hpp +++ b/src/pdal/PyArray.hpp @@ -34,6 +34,7 @@ #pragma once +#include "export.hpp" #include #define NPY_TARGET_VERSION NPY_1_22_API_VERSION @@ -60,7 +61,7 @@ class ArrayIter; using ArrayStreamHandler = std::function; -class PDAL_DLL Array +class PDAL_EXPORT Array { public: using Shape = std::array; @@ -89,7 +90,7 @@ class PDAL_DLL Array }; -class PDAL_DLL ArrayIter +class PDAL_EXPORT ArrayIter { public: ArrayIter(const ArrayIter&) = delete; diff --git a/src/pdal/PyPipeline.hpp b/src/pdal/PyPipeline.hpp index 5233763f..1eed023f 100644 --- a/src/pdal/PyPipeline.hpp +++ b/src/pdal/PyPipeline.hpp @@ -34,6 +34,7 @@ #pragma once +#include "export.hpp" #include #define NPY_TARGET_VERSION NPY_1_22_API_VERSION @@ -55,7 +56,7 @@ PyArrayObject* meshToNumpyArray(const TriangularMesh* mesh); class Array; -class PDAL_DLL PipelineExecutor { +class PDAL_EXPORT PipelineExecutor { public: PipelineExecutor(std::string const& json, std::vector> arrays, int level); virtual ~PipelineExecutor() = default; diff --git a/src/pdal/export.hpp b/src/pdal/export.hpp new file mode 100644 index 00000000..5a6c9aea --- /dev/null +++ b/src/pdal/export.hpp @@ -0,0 +1,44 @@ +/****************************************************************************** +* Copyright (c) 2025, Hobu Inc. (info@hobu.co) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + + +#include + +#ifndef PDAL_EXPORT +# define PDAL_EXPORT PDAL_DLL +#endif + +#ifndef PDAL_DLL +# define PDAL_DLL PDAL_EXPORT +#endif From ca266dc8e541bad841b534468805dc2df2aaf8b4 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:13:04 -0400 Subject: [PATCH 42/59] bump to 3.5.1 to update auto-release --- .github/workflows/release.yml | 3 ++- src/pdal/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 17ab0125..5c9f63d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,10 +23,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Setup micromamba uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge + miniforge-variant: Miniforge3 miniforge-version: latest use-mamba: true python-version: ${{ matrix.python-version }} diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index f311ba7e..2455fd77 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.5.0' +__version__ = '3.5.1' from . import libpdalpython from .drivers import inject_pdal_drivers From 9e2487b9698b3cd89859085ecfcba621b478ded6 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:22:30 -0400 Subject: [PATCH 43/59] update release gha to use trusted publisher --- .github/workflows/release.yml | 58 ++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5c9f63d9..cc664ba8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,37 +10,39 @@ concurrency: cancel-in-progress: true jobs: - pypi-publish: - name: Upload release to PyPI - runs-on: ubuntu-latest - environment: - name: release - url: https://pypi.org/p/pdal-plugins - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - strategy: - fail-fast: true + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup micromamba - uses: conda-incubator/setup-miniconda@v3 + - uses: actions/setup-python@v5 + name: Install Python + with: + python-version: '3.11' + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 with: - miniforge-variant: Miniforge3 - miniforge-version: latest - use-mamba: true - python-version: ${{ matrix.python-version }} - auto-update-conda: true - environment-file: .github/environment.yml - - - name: Install dependencies - shell: bash -l {0} - run: | - python -m pip install build pipx twine - pipx run build --sdist - - - name: Publish package distributions to PyPI - if: github.event_name == 'release' && github.event.action == 'published' - uses: pypa/gh-action-pypi-publish@release/v1 + name: cibw-sdist + path: dist/*.tar.gz + + upload_pypi: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v4 + with: + pattern: cibw-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 From 30546f60a07b8208b8e434b1c505c71b709803f6 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:23:40 -0400 Subject: [PATCH 44/59] update release gha trigger --- .github/workflows/release.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cc664ba8..6c26a204 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,15 @@ name: Release on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/release.yml' + push: + branches: + - main + paths: + - '.github/workflows/release.yml' release: types: - published From b57a1619d6b73e7ed24c244e8e506f9d73039858 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:24:27 -0400 Subject: [PATCH 45/59] fix release gha trigger --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c26a204..013c941b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,7 @@ jobs: path: dist/*.tar.gz upload_pypi: - needs: [build_wheels, build_sdist] + needs: [build_sdist] runs-on: ubuntu-latest environment: pypi permissions: From 4bd3cccf05f406f1defd42a9d9ef779876eeba0d Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:27:51 -0400 Subject: [PATCH 46/59] we need a full environment to build source --- .github/workflows/release.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 013c941b..79197b3c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,10 +26,15 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - name: Install Python + - name: Setup environment + uses: conda-incubator/setup-miniconda@v3 with: - python-version: '3.11' + miniforge-variant: Miniforge3 + miniforge-version: latest + python-version: ${{ matrix.python-version }} + use-mamba: true + auto-update-conda: true + environment-file: .github/environment.yml - name: Build sdist run: pipx run build --sdist From 3689a4224083111ea73e2b06015374df4916606c Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:36:31 -0400 Subject: [PATCH 47/59] environment stuff --- .github/environment.yml | 2 +- .github/workflows/release.yml | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/environment.yml b/.github/environment.yml index 76875d6b..d18b1d3a 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -6,7 +6,7 @@ dependencies: - numpy - compilers - pybind11 - - pdal + - libpdal-core - pytest - meshio - geopandas diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 79197b3c..c532b917 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,6 +36,10 @@ jobs: auto-update-conda: true environment-file: .github/environment.yml + - name: Install pipx + run: | + mamba install -y pipx + - name: Build sdist run: pipx run build --sdist @@ -47,7 +51,7 @@ jobs: upload_pypi: needs: [build_sdist] runs-on: ubuntu-latest - environment: pypi + environment: test permissions: id-token: write if: github.event_name == 'release' && github.event.action == 'published' From 8cf3b042766ca35db81ba37f8d7ae38a0533729a Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:42:39 -0400 Subject: [PATCH 48/59] use env python --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c532b917..942d6e88 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,7 +41,7 @@ jobs: mamba install -y pipx - name: Build sdist - run: pipx run build --sdist + run: python -m pipx run build --sdist - uses: actions/upload-artifact@v4 with: @@ -49,6 +49,7 @@ jobs: path: dist/*.tar.gz upload_pypi: + name: Upload to PyPI needs: [build_sdist] runs-on: ubuntu-latest environment: test From 544173494e451b5c0a288e4beea312897373c376 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:46:59 -0400 Subject: [PATCH 49/59] address scikit-build-core warning for variable replacement --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eca080ee..e1b98235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ sdist.include = [ "src", "CMakeLists.txt" ] -cmake.verbose = false +build.verbose = false logging.level = "ERROR" [tool.scikit-build.metadata.version] From 6b39af581965a74d3c107850c0c3c69ab336c879 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:50:17 -0400 Subject: [PATCH 50/59] try another python build variant --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 942d6e88..e95eb045 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,12 +36,12 @@ jobs: auto-update-conda: true environment-file: .github/environment.yml - - name: Install pipx + - name: Install build run: | - mamba install -y pipx + pip install build - name: Build sdist - run: python -m pipx run build --sdist + run: python -m build --sdist -n - uses: actions/upload-artifact@v4 with: From 6791a880a87e95f7318e99acfb4a10186379c5dd Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 23 Jun 2025 23:54:58 -0400 Subject: [PATCH 51/59] adapt pdal-plugins dist builder --- .github/workflows/release.yml | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e95eb045..8a6ee06e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,37 +31,24 @@ jobs: with: miniforge-variant: Miniforge3 miniforge-version: latest - python-version: ${{ matrix.python-version }} + python-version: '3.12' use-mamba: true auto-update-conda: true environment-file: .github/environment.yml - - name: Install build + - name: Install dependencies + shell: bash -l {0} run: | - pip install build - - - name: Build sdist - run: python -m build --sdist -n + python -m pip install build pipx twine + pipx run build --sdist -Ccmake.define.CMAKE_BUILD_WITH_INSTALL_RPATH=ON - uses: actions/upload-artifact@v4 with: name: cibw-sdist path: dist/*.tar.gz - upload_pypi: - name: Upload to PyPI - needs: [build_sdist] - runs-on: ubuntu-latest - environment: test - permissions: - id-token: write - if: github.event_name == 'release' && github.event.action == 'published' - steps: - - uses: actions/download-artifact@v4 - with: - pattern: cibw-* - path: dist - merge-multiple: true + - name: Publish package distributions to PyPI + if: github.event_name == 'release' && github.event.action == 'published' + uses: pypa/gh-action-pypi-publish@release/v1 - - uses: pypa/gh-action-pypi-publish@release/v1 From c4d070373e44c52d47290e16e2261f53910c0c95 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 3 Sep 2025 09:52:09 -0500 Subject: [PATCH 52/59] adapt testing for PDAL 2.9+ --- README.rst | 2 +- test/test_pipeline.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index cbab9ac1..5bc782f1 100644 --- a/README.rst +++ b/README.rst @@ -422,7 +422,7 @@ USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into ti Requirements ================================================================================ -* PDAL 2.6+ +* PDAL 2.7+ * Python >=3.9 * Pybind11 (eg :code:`pip install pybind11[global]`) * Numpy >= 1.22 (eg :code:`pip install numpy`) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 317c5b69..46e0a18d 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -165,8 +165,11 @@ def test_pipeline(self, filename): """Can we fetch PDAL pipeline string""" r = get_pipeline(filename) r.execute() - assert json.loads(r.pipeline) == { - "pipeline": [ + # filename might be an object in PDAL 2.9+ + # https://github.com/PDAL/PDAL/issues/4751 + + returned = json.loads(r.pipeline) + expected = { "pipeline": [ { "filename": "test/data/1.2-with-color.las", "tag": "readers_las1", @@ -180,6 +183,10 @@ def test_pipeline(self, filename): }, ] } + try: + assert returned['pipeline'][0]['filename'] == "test/data/1.2-with-color.las" + except AttributeError: + assert returned['pipeline'][0]['filename']['path'] == "test/data/1.2-with-color.las" @pytest.mark.parametrize("filename", ["sort.json", "sort.py"]) def test_no_execute(self, filename): From f238c7e48cb9842a5a24256a0015130bc1459669 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 3 Sep 2025 10:30:20 -0500 Subject: [PATCH 53/59] fix trusted publisher --- .github/workflows/release.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a6ee06e..99b22df6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,14 @@ jobs: build_sdist: name: Build source distribution runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/p/pdal-plugins + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + strategy: + fail-fast: true + steps: - uses: actions/checkout@v4 From d0d3782965adf19d5d47f5a6e1d71d9bd0ff2c89 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 3 Sep 2025 10:33:54 -0500 Subject: [PATCH 54/59] bump to 3.5.2 --- src/pdal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index 2455fd77..e2629da2 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.5.1' +__version__ = '3.5.2' from . import libpdalpython from .drivers import inject_pdal_drivers From a9c0dc3c57b5a5a12f7b7d0cf28bd9e79c6837ff Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 3 Sep 2025 11:41:01 -0500 Subject: [PATCH 55/59] bump 3.5.3 for dist issues --- README.rst | 8 +++----- src/pdal/__init__.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 5bc782f1..e8737eb6 100644 --- a/README.rst +++ b/README.rst @@ -254,10 +254,10 @@ To enable this mode, you just need to include the python populate function along The following snippet provides a simple example of how to use a Numpy array as buffer to support writing through PDAL with total control over the maximum amount of memory to use. -.. raw:: html +Example: Streaming the read and write of a very large LAZ file with low memory footprint +......................................................................................... + -
- Example: Streaming the read and write of a very large LAZ file with low memory footprint .. code-block:: python @@ -302,9 +302,7 @@ with total control over the maximum amount of memory to use. print(f"\nWROTE - {count}") -.. raw:: html -
Executing Streamable Pipelines ................................................................................ diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index e2629da2..c67ab5e3 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.5.2' +__version__ = '3.5.3' from . import libpdalpython from .drivers import inject_pdal_drivers From 36417b4104027c9bc612f13f991af272e2702526 Mon Sep 17 00:00:00 2001 From: Kyle Mann Date: Tue, 14 Oct 2025 15:43:52 -0500 Subject: [PATCH 56/59] Adding DTM creation example to README (#196) * Adding DTM creation example to README * fix numbered list --- README.rst | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/README.rst b/README.rst index e8737eb6..ab90fd28 100644 --- a/README.rst +++ b/README.rst @@ -406,6 +406,77 @@ USE-CASE : Take a LiDAR map, create a mesh from the ground points, split into ti buffer.close() +Digital Terrain Model (DTM) Creation Example +................................................................................ + +The following is a script sample that can be used to create a DTM from a PDAL- +readable pointcloud. + +Method: + +#. read point cloud file +#. remove noise +#. clean up invalid values +#. classify ground points using `SMRF `__ +#. write with `GDAL writer `__ + +.. note:: If your pointcloud already has ground classified, you can skip all but + the reader and writer and achieve the same result. + +.. code-block:: python + + import pdal + + pc_path = 'https://github.com/PDAL/data/raw/refs/heads/main/autzen/autzen.laz' + out_file = 'autzen_dtm.tif' + + + # read + reader = pdal.Reader.las(pc_path) + + # remove noisy points + lownoise_filter = pdal.Filter.range( + limits='Classification![7:7]', tag='lownoise' + ) + highnoise_filter = pdal.Filter.range( + limits='Classification![18:]', tag='highnoise' + ) + + # saving incorrectly labeled returns here, some people want this, some don't + prepare_ground = pdal.Filter.assign( + value=[ + 'Classification=0', + 'ReturnNumber=1 WHERE ReturnNumber < 1', + 'NumberOfReturns=1 WHERE NumberOfReturns < 1', + ], + tag='prepare_ground_classifier', + ) + + # classify ground + smrf_classifier = pdal.Filter.smrf(tag='ground_classifier') + + # write with gdal, resolution in feet for autzen + gdal_writer = pdal.Writer.gdal( + filename=out_file, + where='Classification == 2', + data_type='float32', + resolution=10, + output_type='idw', + window_size=3, + pdal_metadata=True, + ) + + # collect pdal stages and execute pipline + pipeline = ( + reader + | lownoise_filter + | highnoise_filter + | prepare_ground + | smrf_classifier + | gdal_writer + ) + pipeline.execute() + .. _`Numpy`: http://www.numpy.org/ .. _`schema`: http://www.pdal.io/dimensions.html From 6b332fcac1de82d8cb0ce28662d1b9b9b5d136e3 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 8 Jul 2026 09:21:46 -0500 Subject: [PATCH 57/59] Test Numpy 2.2+ and update docs (#200) * use uv for build and test, update docs * use build-updates branch of pdal-plugins * fix build-updates usage * set build-updates on checkout * build against main python-plugins * remove setup.py cruft * bump numpy test floor to 2.2, ceiling to 2.5, and test against python 3.14 * some excludes * 3.10 exclusion for 2.5 * readme tweaks --- .github/environment.yml | 12 ++++---- .github/workflows/build.yml | 20 +++++++------ .github/workflows/release.yml | 6 ++-- CMakeLists.txt | 2 +- README.rst | 21 +++++++------ pyproject.toml | 56 ++++++++++++++++++++++++++--------- setup.py.off | 45 ---------------------------- src/pdal/__init__.py | 2 +- 8 files changed, 74 insertions(+), 90 deletions(-) delete mode 100644 setup.py.off diff --git a/.github/environment.yml b/.github/environment.yml index d18b1d3a..348f6abd 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -2,11 +2,13 @@ name: testenv channels: - conda-forge dependencies: - - scikit-build-core - - numpy + - scikit-build-core >=1.0 + - numpy >=1.24 - compilers - - pybind11 + - ninja + - pybind11 >=2.12 - libpdal-core - - pytest - - meshio + - pytest >=8 + - meshio >=5.3 - geopandas + - uv diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc69ec07..7f2c7ec9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,13 +24,15 @@ jobs: fail-fast: true matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-version: ['3.10', '3.11', '3.12', '3.13'] - numpy-version: ['1.24', '2.1'] + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] + numpy-version: ['2.2', '2.5'] exclude: - - python-version: '3.12' - numpy-version: '1.24' - - python-version: '3.13' - numpy-version: '1.24' + - python-version: '3.14' + numpy-version: '2.2' + - python-version: '3.11' + numpy-version: '2.5' + - python-version: '3.10' + numpy-version: '2.5' steps: - name: Check out python-pdal @@ -59,11 +61,12 @@ jobs: - name: Install python-pdal run: | - pip install -vv . --no-deps --no-build-isolation + uv pip install -vv . --no-deps - name: Install python-pdal-plugins working-directory: ./plugins - run: pip install -vv . --no-deps --no-build-isolation + run: | + uv pip install -vv . --no-deps --no-build-isolation - name: Test run: | @@ -75,4 +78,3 @@ jobs: python -m pdal pdal --drivers --debug python -m pytest -v test/ - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99b22df6..dfe8e86b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,11 +44,10 @@ jobs: auto-update-conda: true environment-file: .github/environment.yml - - name: Install dependencies + - name: Build sdist shell: bash -l {0} run: | - python -m pip install build pipx twine - pipx run build --sdist -Ccmake.define.CMAKE_BUILD_WITH_INSTALL_RPATH=ON + uv build --sdist - uses: actions/upload-artifact@v4 with: @@ -59,4 +58,3 @@ jobs: if: github.event_name == 'release' && github.event.action == 'published' uses: pypa/gh-action-pypi-publish@release/v1 - diff --git a/CMakeLists.txt b/CMakeLists.txt index 61610cfb..05c8961d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ endif() # find Python3 find_package(Python3 COMPONENTS Interpreter ${DEVELOPMENT_COMPONENT} NumPy REQUIRED) -# find PDAL. Require 2.1+ +# find PDAL. Require 2.7+ find_package(PDAL 2.7 REQUIRED) # find PyBind11 diff --git a/README.rst b/README.rst index ab90fd28..cb9bebbc 100644 --- a/README.rst +++ b/README.rst @@ -35,13 +35,12 @@ using `scikit-build-core `_ settings: --no-deps \ --no-build-isolation + GitHub ................................................................................ The repository for PDAL's Python extension is available at https://github.com/PDAL/python -Python support released independently from PDAL itself as of PDAL 1.7. - Usage -------------------------------------------------------------------------------- @@ -91,7 +90,7 @@ Stage Objects - A stage is an instance of ``pdal.Reader``, ``pdal.Filter`` or ``pdal.Writer``. - A stage can be instantiated by passing as keyword arguments the options applicable to the respective PDAL stage. For more on PDAL stages and their - options, check the PDAL documentation on `Stage Objects `__. + options, check the PDAL documentation on `Stage Objects `__. - The ``filename`` option of ``Readers`` and ``Writers`` as well as the ``type`` option of ``Filters`` can be passed positionally as the first argument. @@ -417,8 +416,8 @@ Method: #. read point cloud file #. remove noise #. clean up invalid values -#. classify ground points using `SMRF `__ -#. write with `GDAL writer `__ +#. classify ground points using `SMRF `__ +#. write with `GDAL writer `__ .. note:: If your pointcloud already has ground classified, you can skip all but the reader and writer and achieve the same result. @@ -479,11 +478,11 @@ Method: .. _`Numpy`: http://www.numpy.org/ -.. _`schema`: http://www.pdal.io/dimensions.html -.. _`metadata`: http://www.pdal.io/development/metadata.html +.. _`schema`: http://www.pdal.org/dimensions.html +.. _`metadata`: http://www.pdal.org/development/metadata.html .. _`TileDB`: https://tiledb.com/ .. _`TileDB-PDAL integration`: https://docs.tiledb.com/geospatial/pdal -.. _`TileDB writer plugin`: https://pdal.io/stages/writers.tiledb.html +.. _`TileDB writer plugin`: https://pdal.org/stages/writers.tiledb.html .. image:: https://github.com/PDAL/python/workflows/Build/badge.svg :target: https://github.com/PDAL/python/actions?query=workflow%3ABuild @@ -493,6 +492,6 @@ Requirements * PDAL 2.7+ * Python >=3.9 -* Pybind11 (eg :code:`pip install pybind11[global]`) -* Numpy >= 1.22 (eg :code:`pip install numpy`) -* scikit-build-core (eg :code:`pip install scikit-build-core`) +* Pybind11 >= 2.12 (eg :code:`pip install pybind11[global]`) +* Numpy >= 1.24 at runtime and >= 2.0 for isolated builds (eg :code:`pip install numpy`) +* scikit-build-core >= 1.0 (eg :code:`pip install scikit-build-core`) diff --git a/pyproject.toml b/pyproject.toml index e1b98235..52e0e6dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,48 +22,76 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: GIS", ] dependencies = [ - "numpy >= 1.22" + "numpy >= 1.24" ] dynamic = ["version"] [project.optional-dependencies] test = [ - "pandas", - "meshio" + "geopandas", + "meshio >= 5.3", + "pandas >= 2", + "pytest >= 8", ] -[tool.setuptools] -package-dir = {"" = "src"} -zip-safe = false - [project.urls] -homepage = "https://pdal.io" -documentation = "https://pdal.io" +homepage = "https://pdal.org" +documentation = "https://pdal.org" repository = "https://github.com/PDAL/Python" changelog = "https://github.com/PDAL/python/blob/main/README.rst" [build-system] -requires = ["scikit-build-core >= 0.9", "numpy >= 1.22", "pybind11[global]"] +requires = [ + "scikit-build-core >= 1.0", + "numpy >= 2.0", + "pybind11[global] >= 2.12", +] build-backend = "scikit_build_core.build" [tool.scikit-build] +minimum-version = "build-system.requires" build-dir = "build/{wheel_tag}" -sdist.exclude = [".github"] -sdist.cmake = true +sdist.exclude = [ + ".github", + "**/__pycache__/**", + "**/*.pyc", +] +sdist.cmake = false +sdist.inclusion-mode = "explicit" cmake.build-type = "Release" sdist.include = [ + "CHANGES.txt", + "CMakeLists.txt", + "LICENSE.txt", + "pyproject.toml", + "README.rst", "src", - "CMakeLists.txt" +] +wheel.packages = ["src/pdal"] +wheel.exclude = [ + "pdal/*.cpp", + "pdal/*.hpp", + "pdal/export.hpp", + "**/__pycache__/**", + "**/*.pyc", ] build.verbose = false logging.level = "ERROR" -[tool.scikit-build.metadata.version] +[tool.uv] +no-build-isolation = true + +[tool.uv.pip] +no-build-isolation = true + +[[tool.dynamic-metadata]] provider = "scikit_build_core.metadata.regex" +field = "version" input = "src/pdal/__init__.py" diff --git a/setup.py.off b/setup.py.off deleted file mode 100644 index a3fd2a22..00000000 --- a/setup.py.off +++ /dev/null @@ -1,45 +0,0 @@ -from skbuild import setup - -# Get the version from the pdal module -with open("pdal/__init__.py", "r") as fp: - for line in fp: - if line.startswith("__version__"): - version = line.split("=")[1].strip().strip("\"'") - break - else: - raise ValueError("Could not determine Python package version") - -with open("README.rst", "r", encoding="utf-8") as fp: - readme = fp.read() - -with open("CHANGES.txt", "r", encoding="utf-8") as fp: - changes = fp.read() - -setup( - name="pdal", - version=version, - description="Point cloud data processing", - license="BSD", - keywords="point cloud spatial", - author="Howard Butler", - author_email="howard@hobu.co", - maintainer="Howard Butler", - maintainer_email="howard@hobu.co", - url="https://pdal.io", - long_description=readme + "\n\n" + changes, - long_description_content_type="text/x-rst", - packages=["pdal"], - install_requires=["numpy"], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Scientific/Engineering :: GIS", - ], -) diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index c67ab5e3..6a9c4b35 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.5.3' +__version__ = '3.5.4' from . import libpdalpython from .drivers import inject_pdal_drivers From 27a15600a075ffcabb2eafc6a9eb29a58fb4f720 Mon Sep 17 00:00:00 2001 From: Martin Genet <10851655+mrtgenet@users.noreply.github.com> Date: Thu, 9 Jul 2026 23:08:59 +0200 Subject: [PATCH 58/59] Update filename handling in InferableTypeStage for filespec (#198) * Update filename handling in InferableTypeStage for filespec * Validate 'path' in filename for InferableTypeStage Raise ValueError if 'path' is missing in filename dict * Enhance tests for pdal.Reader and pdal.Writer types Added assertions to validate reader and writer types for filespec format. * support and test full filespec definitions * conditionalize FileSpec support to PDAL 2.9+ * check headers for FileSpec --------- Co-authored-by: Howard Butler --- src/pdal/pipeline.py | 40 ++++++++++++++++++++++++++++++++++++++-- test/test_pipeline.py | 15 ++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/pdal/pipeline.py b/src/pdal/pipeline.py index 60a181c0..4310b232 100644 --- a/src/pdal/pipeline.py +++ b/src/pdal/pipeline.py @@ -33,6 +33,13 @@ LogLevelFromPDAL = {v: k for k, v in LogLevelToPDAL.items()} +HaveFileSpecSupport = False +if libpdalpython.getInfo().major == 2 and \ + libpdalpython.getInfo().minor >= 9 or \ + libpdalpython.getInfo().major > 2: + HaveFileSpecSupport = True + + class Pipeline(libpdalpython.Pipeline): def __init__( self, @@ -217,7 +224,25 @@ def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: class InferableTypeStage(Stage): def __init__(self, filename: Optional[str] = None, **options: Any): if filename: - options["filename"] = filename + if isinstance(filename, dict): + if "path" not in filename: + raise ValueError(f"'path' is missing in the provided filespec: {filename}") + if HaveFileSpecSupport: + options["filename"] = filename + else: + # log that we can't pass FileSpec to PDAL + # because the library version is too old + msg = "PDAL library version is too old for FileSpec support. " \ + "Defaulting to using filename only" + logging.info(msg) + options["filename"] = filename["path"] + + else: + if HaveFileSpecSupport: + filespec = {'path':str(filename)} + options["filename"] = filespec + else: + options["filename"] = filename super().__init__(**options) @property @@ -226,7 +251,18 @@ def type(self) -> str: return super().type except KeyError: filename = self._options.get("filename") - return str(self._infer_type(filename) if filename else "") + if isinstance(filename, dict): + if "path" not in filename: + raise ValueError(f"'path' is missing in the provided filespec: {filename}") + if HaveFileSpecSupport: + path = filename.get('path') + else: + path = filename + + else: + path = str(filename) + + return str(self._infer_type(path) if filename else "") _infer_type = staticmethod(lambda filename: "") diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 46e0a18d..9bb3ba89 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -3,7 +3,6 @@ import os import sys -from typing import List from itertools import product import numpy as np import pytest @@ -284,6 +283,20 @@ def test_infer_stage_type(self): assert pdal.Writer("foo.xxx").type == "" assert pdal.Reader().type == "" assert pdal.Writer().type == "" + assert pdal.Reader({"path": "foo.las"}).type == "readers.las" + assert pdal.Writer({"path": "foo.las"}).type == "writers.las" + assert pdal.Reader({"path": "foo.xxx"}).type == "" + assert pdal.Writer({"path": "foo.xxx"}).type == "" + assert pdal.Reader({}).type == "" + assert pdal.Writer({}).type == "" + + def test_filespec(self): + """Can transit filespecs""" + spec = {'path':'junk.las', 'headers':{'header1':'header_1', 'header2':'header_2'}, 'query':{'query1':'query_1', 'query2':'query_2'}} + assert pdal.Reader(spec).type == "readers.las" + assert pdal.Reader(spec).options['filename']['path'] == "junk.las" + assert pdal.Reader(spec).options['filename']['headers']['header2'] == "header_2" + assert pdal.Writer("foo.las").type == "writers.las" def test_streamable(self): """Can we distinguish streamable from non-streamable stages and pipeline""" From c963beb9bcf1830cc8b81fa19e1deb14e1aa1906 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 31 Jul 2026 10:05:42 -0500 Subject: [PATCH 59/59] add a timing parameter to enable logtiming option (#201) * add a an timing parameter to enable logtiming option in Pipeline invocations * add note to readme * bump to 3.5.5 --- README.rst | 12 ++++++++++++ src/pdal/PyPipeline.cpp | 4 ++-- src/pdal/PyPipeline.hpp | 2 +- src/pdal/StreamableExecutor.cpp | 3 ++- src/pdal/StreamableExecutor.hpp | 1 + src/pdal/__init__.py | 2 +- src/pdal/libpdalpython.cpp | 12 +++++++++--- src/pdal/pipeline.py | 22 +++++++++++++++++++--- test/test_pipeline.py | 29 ++++++++++++++++++++++++++++- 9 files changed, 75 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index cb9bebbc..e352ca4b 100644 --- a/README.rst +++ b/README.rst @@ -73,6 +73,18 @@ sorts it by the ``X`` dimension: metadata = pipeline.metadata log = pipeline.log +Pass ``timing=True`` when creating a pipeline to include PDAL timing +information in ``pipeline.log`` after execution: + +.. code-block:: python + + import logging + import pdal + + pipeline = pdal.Pipeline(json, loglevel=logging.DEBUG, timing=True) + count = pipeline.execute() + timing_log = pipeline.log + Programmatic Pipeline Construction ................................................................................ diff --git a/src/pdal/PyPipeline.cpp b/src/pdal/PyPipeline.cpp index 7f295273..64abe7a7 100644 --- a/src/pdal/PyPipeline.cpp +++ b/src/pdal/PyPipeline.cpp @@ -56,12 +56,12 @@ void CountPointTable::reset() PipelineExecutor::PipelineExecutor( - std::string const& json, std::vector> arrays, int level) + std::string const& json, std::vector> arrays, int level, bool timing) { if (level < 0 || level > 8) throw pdal_error("log level must be between 0 and 8!"); - LogPtr log(Log::makeLog("pypipeline", &m_logStream)); + LogPtr log(Log::makeLog("pypipeline", &m_logStream, timing)); log->setLevel(static_cast(level)); m_manager.setLog(log); diff --git a/src/pdal/PyPipeline.hpp b/src/pdal/PyPipeline.hpp index 1eed023f..45fa241e 100644 --- a/src/pdal/PyPipeline.hpp +++ b/src/pdal/PyPipeline.hpp @@ -58,7 +58,7 @@ class Array; class PDAL_EXPORT PipelineExecutor { public: - PipelineExecutor(std::string const& json, std::vector> arrays, int level); + PipelineExecutor(std::string const& json, std::vector> arrays, int level, bool timing); virtual ~PipelineExecutor() = default; point_count_t execute(pdal::StringList allowedDims); diff --git a/src/pdal/StreamableExecutor.cpp b/src/pdal/StreamableExecutor.cpp index 5fa01931..aad2d9d2 100644 --- a/src/pdal/StreamableExecutor.cpp +++ b/src/pdal/StreamableExecutor.cpp @@ -186,10 +186,11 @@ char *PythonPointTable::getPoint(PointId idx) StreamableExecutor::StreamableExecutor(std::string const& json, std::vector> arrays, int level, + bool timing, point_count_t chunkSize, int prefetch, pdal::StringList allowedDims) - : PipelineExecutor(json, arrays, level) + : PipelineExecutor(json, arrays, level, timing) , m_table(chunkSize, prefetch) , m_exc(nullptr) { diff --git a/src/pdal/StreamableExecutor.hpp b/src/pdal/StreamableExecutor.hpp index f565c8ee..bb852755 100644 --- a/src/pdal/StreamableExecutor.hpp +++ b/src/pdal/StreamableExecutor.hpp @@ -80,6 +80,7 @@ class StreamableExecutor : public PipelineExecutor StreamableExecutor(std::string const& json, std::vector> arrays, int level, + bool timing, point_count_t chunkSize, int prefetch, pdal::StringList allowedDim); diff --git a/src/pdal/__init__.py b/src/pdal/__init__.py index 6a9c4b35..fb790e0d 100644 --- a/src/pdal/__init__.py +++ b/src/pdal/__init__.py @@ -1,5 +1,5 @@ __all__ = ["Pipeline", "Stage", "Reader", "Filter", "Writer", "dimensions", "info"] -__version__ = '3.5.4' +__version__ = '3.5.5' from . import libpdalpython from .drivers import inject_pdal_drivers diff --git a/src/pdal/libpdalpython.cpp b/src/pdal/libpdalpython.cpp index 09118fbf..9ce7a939 100644 --- a/src/pdal/libpdalpython.cpp +++ b/src/pdal/libpdalpython.cpp @@ -186,7 +186,7 @@ namespace pdal { std::unique_ptr iterator(int chunk_size, int prefetch, pdal::StringList allowedDims) { return std::unique_ptr(new PipelineIterator( - getJson(), _inputs, _loglevel, chunk_size, prefetch, allowedDims + getJson(), _inputs, _loglevel, _timing, chunk_size, prefetch, allowedDims )); } @@ -214,6 +214,10 @@ namespace pdal { void setLogLevel(int level) { _loglevel = level; delExecutor(); } + bool getTiming() { return _timing; } + + void setTiming(bool timing) { _timing = timing; delExecutor(); } + std::string getLog() { return getExecutor()->getLog(); } std::string getPipeline() { return getExecutor()->getPipeline(); } @@ -291,14 +295,15 @@ namespace pdal { // does for all of the other methods it knows about py::gil_scoped_acquire acquire; if (!_executor) - _executor.reset(new PipelineExecutor(getJson(), _inputs, _loglevel)); + _executor.reset(new PipelineExecutor(getJson(), _inputs, _loglevel, _timing)); return _executor.get(); } private: std::unique_ptr _executor; std::vector> _inputs; - int _loglevel; + int _loglevel = 0; + bool _timing = false; }; @@ -324,6 +329,7 @@ namespace pdal { .def("iterator", &Pipeline::iterator, "chunk_size"_a=10000, "prefetch"_a=0, py::arg("allowed_dims") =py::list()) .def_property("inputs", nullptr, &Pipeline::setInputs) .def_property("loglevel", &Pipeline::getLoglevel, &Pipeline::setLogLevel) + .def_property("timing", &Pipeline::getTiming, &Pipeline::setTiming) .def_property_readonly("log", &Pipeline::getLog) .def_property_readonly("schema", &Pipeline::getSchema) .def_property_readonly("srswkt2", &Pipeline::getSrsWKT2) diff --git a/src/pdal/pipeline.py b/src/pdal/pipeline.py index 4310b232..889bc075 100644 --- a/src/pdal/pipeline.py +++ b/src/pdal/pipeline.py @@ -49,6 +49,8 @@ def __init__( json: Optional[str] = None, dataframes: Sequence[DataFrame] = (), stream_handlers: Sequence[Callable[[], int]] = (), + *, + timing: bool = False, ): if json: @@ -75,6 +77,7 @@ def __init__( self.inputs = [(a, None) for a in arrays] self.loglevel = loglevel + self.timing = timing def __getstate__(self): state = self.pipeline @@ -104,6 +107,14 @@ def loglevel(self, value: int) -> None: # super() property setter is not supported libpdalpython.Pipeline.loglevel.__set__(self, loglevel) + @property + def timing(self) -> bool: + return super().timing + + @timing.setter + def timing(self, value: bool) -> None: + libpdalpython.Pipeline.timing.__set__(self, bool(value)) + def __ior__(self, other: Union[Stage, Pipeline]) -> Pipeline: if isinstance(other, Stage): self._stages.append(other) @@ -124,7 +135,7 @@ def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return new def __copy__(self) -> Pipeline: - clone = self.__class__(loglevel=self.loglevel) + clone = self.__class__(loglevel=self.loglevel, timing=self.timing) clone._copy_inputs(self) clone |= self return clone @@ -214,8 +225,13 @@ def inputs(self) -> List[Union[Stage, str]]: def options(self) -> Dict[str, Any]: return dict(self._options) - def pipeline(self, *arrays: np.ndarray, loglevel: int = logging.ERROR) -> Pipeline: - return Pipeline((self,), arrays, loglevel) + def pipeline( + self, + *arrays: np.ndarray, + loglevel: int = logging.ERROR, + timing: bool = False, + ) -> Pipeline: + return Pipeline((self,), arrays, loglevel=loglevel, timing=timing) def __or__(self, other: Union[Stage, Pipeline]) -> Pipeline: return Pipeline((self, other)) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 9bb3ba89..3eac09e9 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -1,6 +1,7 @@ import json import logging import os +import re import sys from itertools import product @@ -53,6 +54,21 @@ def test_construction(self, filename): assert isinstance(p, pdal.Pipeline) assert len(p.stages) == 2 + def test_timing_is_optional_public_api(self): + with open(os.path.join(DATADIRECTORY, "chip.json"), "r") as f: + pipeline_json = f.read() + + p = pdal.Pipeline(None, (), logging.ERROR, pipeline_json) + assert p.timing is False + assert p.execute() == 1065 + + with pytest.raises(TypeError): + pdal.Pipeline(None, (), logging.ERROR, pipeline_json, (), (), True) + + reader = pdal.Reader(os.path.join(DATADIRECTORY, "1.2-with-color.las")) + assert reader.pipeline().timing is False + assert reader.pipeline(timing=True).timing is True + @pytest.mark.parametrize( "pipeline", [ @@ -338,6 +354,7 @@ def test_logging(self, filename): """Can we fetch log output""" r = get_pipeline(filename) assert r.loglevel == logging.ERROR + assert r.timing is False assert r.log == "" for loglevel in logging.CRITICAL, -1: @@ -356,6 +373,17 @@ def test_logging(self, filename): assert "(pypipeline Debug) Executing pipeline in standard mode" in r.log assert "(pypipeline writers.las Debug)" in r.log + def test_logging_timing(self): + """Can we fetch log output decorated with timing information""" + with open(os.path.join(DATADIRECTORY, "chip.json"), "r") as f: + r = pdal.Pipeline(f.read(), loglevel=logging.DEBUG, timing=True) + + assert r.timing is True + count = r.execute() + assert count == 1065 + assert re.search(r"\(pypipeline readers\.las Debug [0-9.]+\)", r.log) + assert re.search(r"\(pypipeline Debug [0-9.]+\) Executing pipeline in standard mode", r.log) + @pytest.mark.skipif( not hasattr(pdal.Filter, "python"), reason="filters.python PDAL plugin is not available", @@ -904,4 +932,3 @@ def invalid_stream_handler(): with pytest.raises(RuntimeError, match=f"Stream chunk size not in the range of array length: {invalid_chunk_size}"): p.execute() -