From bd152487b121a4aeb7065bfab901d094f846a4f6 Mon Sep 17 00:00:00 2001 From: sedot42 Date: Mon, 29 Oct 2018 09:53:51 +0100 Subject: [PATCH 001/343] correct typo in README.rst, fix #12 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7a228bb5..5bac79b4 100644 --- a/README.rst +++ b/README.rst @@ -35,7 +35,7 @@ sorts it by the ``X`` dimension: import pdal pipeline = pdal.Pipeline(json) pipeline.validate() # check if our JSON and options were good - pipeline.loglevel = 9 #really noisy + pipeline.loglevel = 8 #really noisy count = pipeline.execute() arrays = pipeline.arrays metadata = pipeline.metadata From 75b9683ab48f282a3425ae3b3369aa670f6ea2eb Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 29 Oct 2018 11:28:37 -0500 Subject: [PATCH 002/343] test multiple arrays properly --- test/test_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 5b6516be..4ec26290 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -132,7 +132,6 @@ def test_merged_arrays(self): data = np.load(os.path.join(DATADIRECTORY, 'perlin.npy')) arrays = [data, data, data] - arrays = [data] json = self.fetch_json('chip.json') chip =u"""{ @@ -148,10 +147,11 @@ def test_merged_arrays(self): p.loglevel = 8 count = p.execute() arrays = p.arrays - self.assertEqual(len(arrays), 1) + self.assertEqual(len(arrays), 3) data = arrays[0] self.assertEqual(len(data), 1836) + self.assertEqual(sum([len(i) for i in arrays]), 3*1836) class TestDimensions(PDALTest): def test_fetch_dimensions(self): From 2d899a0d1de4652872de6d54f4f8708dc05e366a Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 31 Oct 2018 10:32:32 -0500 Subject: [PATCH 003/343] increment version to 2.1.0rc1 --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 50aea0e7..112370af 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.0 \ No newline at end of file +2.1.0rc1 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index 0d42ee08..8b7a645c 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.0' +__version__='2.1.0rc1' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index 1d4a56b5..c8086432 100644 --- a/setup.py +++ b/setup.py @@ -178,7 +178,7 @@ def get_pdal_config(option): setup_args = dict( name = 'PDAL', version = str(module_version), - requires = ['Python (>=2.7)', ], + requires = ['Python (>=2.7)', 'Numpy'], description = 'Point cloud data processing', license = 'BSD', keywords = 'point cloud spatial', @@ -203,7 +203,7 @@ def get_pdal_config(option): 'Topic :: Scientific/Engineering :: GIS', ], cmdclass = {}, - install_requires = ['numpy', 'packaging'], + install_requires = ['numpy', 'packaging', 'cython'], ) setup(ext_modules=extensions, **setup_args) From bc88cca063f37e471538da31432b62134b93a999 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 31 Oct 2018 16:26:47 -0500 Subject: [PATCH 004/343] * Support fetching version information from PDAL * Allow Python extension to be out of sync with PDAL version (tests, * libraries, etc) --- pdal/PyPipeline.cpp | 3 +++ pdal/__init__.py | 13 +++++++++++++ pdal/libpdalpython.pyx | 26 ++++++++++++++++++++++++++ setup.py | 21 +++++++++++++++------ test/test_pipeline.py | 8 +++++++- 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index a7096f6e..27c377f9 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -47,6 +47,7 @@ #include "PyArray.hpp" #include +#include #include #include @@ -78,6 +79,7 @@ Pipeline::Pipeline(std::string const& json, std::vector arrays) if (!r) throw pdal::pdal_error("pipeline had no stages!"); +#if PDAL_VERSION_MAJOR > 1 || PDAL_VERSION_MINOR >=8 int counter = 1; for (auto array: arrays) { @@ -102,6 +104,7 @@ Pipeline::Pipeline(std::string const& json, std::vector arrays) counter++; } +#endif manager.validateStageOptions(); } diff --git a/pdal/__init__.py b/pdal/__init__.py index 8b7a645c..9f760d76 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -3,3 +3,16 @@ from .pipeline import Pipeline from .array import Array from .dimension import dimensions + +from pdal.libpdalpython import getVersionString, getVersionMajor, getVersionMinor, getVersionPatch, getSha1, getDebugInformation, getPluginInstallPath + +class Info(object): + version = getVersionString() + major = getVersionMajor() + minor = getVersionMinor() + patch = getVersionPatch() + debug = getDebugInformation() + sha1 = getSha1() + plugin = getPluginInstallPath() + +info = Info() diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index 9a02972f..a33f182c 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -12,6 +12,30 @@ np.import_array() from cpython cimport PyObject, Py_INCREF from cython.operator cimport dereference as deref, preincrement as inc +cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": + cdef int versionMajor() except + + cdef int versionMinor() except + + cdef int versionPatch() except + + cdef string sha1() except+ + cdef string debugInformation() except+ + cdef string pluginInstallPath() except+ + cdef string versionString() except+ + +def getVersionString(): + return versionString() + +def getVersionMajor(): + return versionMajor() +def getVersionMinor(): + return versionMinor() +def getVersionPatch(): + return versionPatch() +def getSha1(): + return sha1() +def getDebugInformation(): + return debugInformation() +def getPluginInstallPath(): + return pluginInstallPath() cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: @@ -32,6 +56,8 @@ cdef extern from "PyPipeline.hpp" namespace "libpdalpython": int getLogLevel() void setLogLevel(int) + + cdef class PyArray: cdef Array *thisptr def __cinit__(self, object array): diff --git a/setup.py b/setup.py index c8086432..4bb1cf3d 100644 --- a/setup.py +++ b/setup.py @@ -108,13 +108,9 @@ def get_pdal_config(option): extra_link_args = [] extra_compile_args = [] -if os.name in ['nt']: - library_dirs = ['c:/OSGeo4W64/lib'] - libraries = ['pdalcpp','pdal_plugin_reader_numpy','pdal_util','ws2_32'] - extra_compile_args = ['/DNOMINMAX',] - from setuptools.extension import Extension as DistutilsExtension +PDALVERSION = None if pdal_config and "clean" not in sys.argv: # Collect other options from PDAL try: @@ -134,6 +130,7 @@ def get_pdal_config(option): # older versions of pdal-config do not include --python-version switch except ValueError: pass + PDALVERSION = Version(get_pdal_config('--version')) separator = ':' if os.name in ['nt']: @@ -162,7 +159,19 @@ def get_pdal_config(option): if os.name != 'nt': extra_compile_args += ['-g','-O0'] -libraries.append('pdal_plugin_reader_numpy') +if os.name in ['nt']: + if os.environ['OSGEO4W_ROOT']: + library_dirs = ['c:/OSGeo4W64/lib'] + libraries = ['pdalcpp','pdal_util','ws2_32'] + if PDALVERSION >= Version('1.8'): + libraries.append('pdal_plugin_reader_numpy') + extra_compile_args = ['/DNOMINMAX',] + + +# readers.numpy doesn't exist until PDAL 1.8 +if PDALVERSION >= Version('1.8'): + libraries.append('pdal_plugin_reader_numpy') + sources=['pdal/libpdalpython'+ext, "pdal/PyPipeline.cpp" ] extensions = [DistutilsExtension("*", sources, diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 4ec26290..96414a08 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -2,6 +2,7 @@ import pdal import os import numpy as np +from packaging.version import Version DATADIRECTORY = "./test/data" @@ -129,6 +130,8 @@ class TestArrayLoad(PDALTest): def test_merged_arrays(self): """Can we load data from a a list of arrays to PDAL""" + if Version(pdal.info.version) < Version('1.8'): + return True data = np.load(os.path.join(DATADIRECTORY, 'perlin.npy')) arrays = [data, data, data] @@ -157,7 +160,10 @@ class TestDimensions(PDALTest): def test_fetch_dimensions(self): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - self.assertEqual(len(dims), 72) + if Version(pdal.info.version) < Version('1.8'): + self.assertEqual(len(dims), 71) + else: + self.assertEqual(len(dims), 72) def test_suite(): return unittest.TestSuite( From 74005676ba8fdbf03869bd8e0a488c920c81e177 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 1 Nov 2018 07:59:37 -0500 Subject: [PATCH 005/343] 2.1.0rc3 to not run tests when a test file doesn't exist --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- test/test_pipeline.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 112370af..0baf8a27 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.0rc1 \ No newline at end of file +2.1.0rc2 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index 9f760d76..8238a5b5 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.0rc1' +__version__='2.1.0rc3' from .pipeline import Pipeline from .array import Array diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 96414a08..35710360 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -128,6 +128,8 @@ def test_merged_arrays(self): # class TestArrayLoad(PDALTest): + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'perlin.npy')), + "missing test data") def test_merged_arrays(self): """Can we load data from a a list of arrays to PDAL""" if Version(pdal.info.version) < Version('1.8'): From 6661d76aaf7102f77c247eef243361309a0a0914 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 15 Nov 2018 08:55:07 -0600 Subject: [PATCH 006/343] attempt to fix gcc ABI --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- setup.py | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 0baf8a27..7c327287 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.0rc2 \ No newline at end of file +2.1.1 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index 8238a5b5..4382b743 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.0rc3' +__version__='2.1.1' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index 4bb1cf3d..56184f2c 100644 --- a/setup.py +++ b/setup.py @@ -148,9 +148,6 @@ def get_pdal_config(option): include_dirs.append(numpy.get_include()) -if os.name != 'nt': - extra_compile_args = ['-std=c++11','-Wno-unknown-pragmas'] - if platform.system() == 'Darwin': extra_link_args.append('-Wl,-rpath,'+library_dirs[0]) @@ -167,6 +164,10 @@ def get_pdal_config(option): libraries.append('pdal_plugin_reader_numpy') extra_compile_args = ['/DNOMINMAX',] +if 'linux' in sys.platform or 'linux2' in sys.platform: + # try to ensure the ABI + if 'GCC' in sys.version: + extra_compile_args += '-D_GLIBCXX_USE_CXX11_ABI=0' # readers.numpy doesn't exist until PDAL 1.8 if PDALVERSION >= Version('1.8'): From 8b5e13de204fbaf62ad14209082546dd036d44ab Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 15 Nov 2018 10:35:00 -0600 Subject: [PATCH 007/343] git fix #18 - ABI args --- pdal/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pdal/__init__.py b/pdal/__init__.py index 4382b743..a72d6cfe 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.1' +__version__='2.1.2' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index 56184f2c..abef0d39 100644 --- a/setup.py +++ b/setup.py @@ -167,7 +167,7 @@ def get_pdal_config(option): if 'linux' in sys.platform or 'linux2' in sys.platform: # try to ensure the ABI if 'GCC' in sys.version: - extra_compile_args += '-D_GLIBCXX_USE_CXX11_ABI=0' + extra_compile_args += ['-D_GLIBCXX_USE_CXX11_ABI=0'] # readers.numpy doesn't exist until PDAL 1.8 if PDALVERSION >= Version('1.8'): From b787cfb3eca4191d7c4c1fe803d161ca604843be Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 15 Nov 2018 11:15:17 -0600 Subject: [PATCH 008/343] ABI args for conda only --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- setup.py | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 7c327287..8f9174b4 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.1 \ No newline at end of file +2.1.2 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index a72d6cfe..010b2d82 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.2' +__version__='2.1.3' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index abef0d39..10eaa271 100644 --- a/setup.py +++ b/setup.py @@ -165,9 +165,10 @@ def get_pdal_config(option): extra_compile_args = ['/DNOMINMAX',] if 'linux' in sys.platform or 'linux2' in sys.platform: - # try to ensure the ABI if 'GCC' in sys.version: - extra_compile_args += ['-D_GLIBCXX_USE_CXX11_ABI=0'] + # try to ensure the ABI for Conda GCC 4.8 + if '4.8' in sys.version: + extra_compile_args += ['-D_GLIBCXX_USE_CXX11_ABI=0'] # readers.numpy doesn't exist until PDAL 1.8 if PDALVERSION >= Version('1.8'): From dc7089e6aa5618e07c741a07f8e6cfe127376551 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 20 Nov 2018 09:15:39 -0600 Subject: [PATCH 009/343] fix OSX/clang building --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- setup.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 8f9174b4..abae0d9a 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.2 \ No newline at end of file +2.1.3 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index 010b2d82..463749cd 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.3' +__version__='2.1.4' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index 10eaa271..dcd6c5a7 100644 --- a/setup.py +++ b/setup.py @@ -169,6 +169,9 @@ def get_pdal_config(option): # try to ensure the ABI for Conda GCC 4.8 if '4.8' in sys.version: extra_compile_args += ['-D_GLIBCXX_USE_CXX11_ABI=0'] +elif 'darwin' in sys.platform: + if 'Clang' in sys.version: + extra_compile_args += ['-std=c++11', '-Wno-unknown-pragmas'] # readers.numpy doesn't exist until PDAL 1.8 if PDALVERSION >= Version('1.8'): From 5d25746f69dab0bbb4ed3458fd1726c1520adef5 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 20 Nov 2018 09:25:29 -0600 Subject: [PATCH 010/343] c++11 on osx --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- setup.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index abae0d9a..c5864dcf 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.3 \ No newline at end of file +2.1.5 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index 463749cd..cc4988b3 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.4' +__version__='2.1.5' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index dcd6c5a7..d9cee0a2 100644 --- a/setup.py +++ b/setup.py @@ -170,8 +170,7 @@ def get_pdal_config(option): if '4.8' in sys.version: extra_compile_args += ['-D_GLIBCXX_USE_CXX11_ABI=0'] elif 'darwin' in sys.platform: - if 'Clang' in sys.version: - extra_compile_args += ['-std=c++11', '-Wno-unknown-pragmas'] + extra_compile_args += ['-std=c++11', '-Wno-unknown-pragmas'] # readers.numpy doesn't exist until PDAL 1.8 if PDALVERSION >= Version('1.8'): From ca83edaff9e16592056db90dd34e1dcc3af79e3a Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 20 Nov 2018 12:55:12 -0600 Subject: [PATCH 011/343] bump to 2.1.6 to fix windows issues --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- setup.py | 19 +++++++++++++------ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index c5864dcf..b6da5122 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.5 \ No newline at end of file +2.1.6 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index cc4988b3..d93314be 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.5' +__version__='2.1.6' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index d9cee0a2..db535820 100644 --- a/setup.py +++ b/setup.py @@ -156,12 +156,21 @@ def get_pdal_config(option): if os.name != 'nt': extra_compile_args += ['-g','-O0'] +# readers.numpy doesn't exist until PDAL 1.8 +if PDALVERSION >= Version('1.8'): + libraries.append('pdal_plugin_reader_numpy') + if os.name in ['nt']: - if os.environ['OSGEO4W_ROOT']: - library_dirs = ['c:/OSGeo4W64/lib'] + if os.environ.get('OSGEO4W_ROOT'): + library_dirs = ['c:/%s/lib' % os.environ.get('OSGEO4W_ROOT')] + if os.environ.get('CONDA_PREFIX'): + prefix=os.path.expandvars('%CONDA_PREFIX%') + library_dirs = ['%s\Library\lib' % prefix] + libraries = ['pdalcpp','pdal_util','ws2_32'] if PDALVERSION >= Version('1.8'): - libraries.append('pdal_plugin_reader_numpy') + libraries.append('libpdal_plugin_reader_numpy') + extra_compile_args = ['/DNOMINMAX',] if 'linux' in sys.platform or 'linux2' in sys.platform: @@ -172,9 +181,7 @@ def get_pdal_config(option): elif 'darwin' in sys.platform: extra_compile_args += ['-std=c++11', '-Wno-unknown-pragmas'] -# readers.numpy doesn't exist until PDAL 1.8 -if PDALVERSION >= Version('1.8'): - libraries.append('pdal_plugin_reader_numpy') + sources=['pdal/libpdalpython'+ext, "pdal/PyPipeline.cpp" ] extensions = [DistutilsExtension("*", From 47258385ea8ab2f837ba035fdabc352ec2808275 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 20 Nov 2018 13:13:22 -0600 Subject: [PATCH 012/343] docker changed --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b2162cb4..299bc849 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ sudo: required services: docker before_install: - - docker pull pdal/ubuntu:master + - docker pull pdal/pdal:master script: - - docker run -v $TRAVIS_BUILD_DIR:/pdal -t pdal/ubuntu:master /bin/sh -c "/pdal/scripts/travis/script.sh" + - docker run -v $TRAVIS_BUILD_DIR:/pdal -t pdal/pdal:master /bin/sh -c "/pdal/scripts/travis/script.sh" From fdf228398ce0843d3f14620023518786b07552c0 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 20 Nov 2018 13:16:47 -0600 Subject: [PATCH 013/343] latest docker, not master --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 299bc849..5bc813b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ sudo: required services: docker before_install: - - docker pull pdal/pdal:master + - docker pull pdal/pdal:latest script: - - docker run -v $TRAVIS_BUILD_DIR:/pdal -t pdal/pdal:master /bin/sh -c "/pdal/scripts/travis/script.sh" + - docker run -v $TRAVIS_BUILD_DIR:/pdal -t pdal/pdal:latest /bin/sh -c "/pdal/scripts/travis/script.sh" From 95c26e60230fecdbe82e023f9a6e09ad0c20b2a3 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 21 Nov 2018 10:03:54 -0600 Subject: [PATCH 014/343] try using c++11 for everything, not just darwin --- VERSION.txt | 2 +- pdal/__init__.py | 2 +- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index b6da5122..9671f9a9 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.6 \ No newline at end of file +2.1.7 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index d93314be..cc4535c7 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.6' +__version__='2.1.7' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index db535820..4cc676f5 100644 --- a/setup.py +++ b/setup.py @@ -166,7 +166,7 @@ def get_pdal_config(option): if os.environ.get('CONDA_PREFIX'): prefix=os.path.expandvars('%CONDA_PREFIX%') library_dirs = ['%s\Library\lib' % prefix] - + libraries = ['pdalcpp','pdal_util','ws2_32'] if PDALVERSION >= Version('1.8'): libraries.append('libpdal_plugin_reader_numpy') @@ -178,7 +178,7 @@ def get_pdal_config(option): # try to ensure the ABI for Conda GCC 4.8 if '4.8' in sys.version: extra_compile_args += ['-D_GLIBCXX_USE_CXX11_ABI=0'] -elif 'darwin' in sys.platform: +else: extra_compile_args += ['-std=c++11', '-Wno-unknown-pragmas'] From 37328e3d79ef8741a347f4a0a8de738ec3d1be17 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 21 Nov 2018 13:13:51 -0600 Subject: [PATCH 015/343] 2.1.8 --- pdal/__init__.py | 2 +- setup.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pdal/__init__.py b/pdal/__init__.py index cc4535c7..c287bae6 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.7' +__version__='2.1.8' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index 4cc676f5..2a3fed2e 100644 --- a/setup.py +++ b/setup.py @@ -173,13 +173,12 @@ def get_pdal_config(option): extra_compile_args = ['/DNOMINMAX',] -if 'linux' in sys.platform or 'linux2' in sys.platform: +if 'linux' in sys.platform or 'linux2' in sys.platform or 'darwin' in sys.platform: + extra_compile_args += ['-std=c++11', '-Wno-unknown-pragmas'] if 'GCC' in sys.version: # try to ensure the ABI for Conda GCC 4.8 if '4.8' in sys.version: extra_compile_args += ['-D_GLIBCXX_USE_CXX11_ABI=0'] -else: - extra_compile_args += ['-std=c++11', '-Wno-unknown-pragmas'] From 083c93a8fc0a9d1e0fd91b26ad82a32c6286dd5f Mon Sep 17 00:00:00 2001 From: Bas Couwenberg Date: Sat, 30 Mar 2019 16:02:16 +0100 Subject: [PATCH 016/343] Don't append library in clean target. PDALVERSION is not set in the clean target. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2a3fed2e..8bd2fd3e 100644 --- a/setup.py +++ b/setup.py @@ -157,7 +157,7 @@ def get_pdal_config(option): extra_compile_args += ['-g','-O0'] # readers.numpy doesn't exist until PDAL 1.8 -if PDALVERSION >= Version('1.8'): +if PDALVERSION is not None and PDALVERSION >= Version('1.8'): libraries.append('pdal_plugin_reader_numpy') if os.name in ['nt']: From c5a7d50ea34988407a85e9b34a7cf98dd53bb774 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Mon, 1 Apr 2019 15:42:00 -0500 Subject: [PATCH 017/343] Python3 now, not Python2 for pdal/pdal:latest --- scripts/travis/script.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/travis/script.sh b/scripts/travis/script.sh index a1891d38..84f837c6 100755 --- a/scripts/travis/script.sh +++ b/scripts/travis/script.sh @@ -1,5 +1,5 @@ -apt-get update -y; apt-get install python-pip -y -pip install numpy packaging cython +apt-get update -y; apt-get install python3-pip -y +pip3 install numpy packaging cython cd /pdal/ -python setup.py build -python setup.py test +python3 setup.py build +python3 setup.py test From 5ca8b4f07b3c6d36bd9756d8d1a2e1bb72ee24d4 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 2 Apr 2019 13:50:37 -0500 Subject: [PATCH 018/343] Implement AppVeyor for PDAL/Python repository (#25) * add appveyor config * point to scripts * need cython and packaging * add appveyor badge --- .appveyor.yml | 33 +++++++++++++++++++++++++++++++++ README.rst | 3 +++ scripts/appveyor/build.cmd | 7 +++++++ scripts/appveyor/config.cmd | 16 ++++++++++++++++ scripts/appveyor/test.cmd | 3 +++ 5 files changed, 62 insertions(+) create mode 100644 .appveyor.yml create mode 100644 scripts/appveyor/build.cmd create mode 100644 scripts/appveyor/config.cmd create mode 100644 scripts/appveyor/test.cmd diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 00000000..0effb3f8 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,33 @@ +version: 1.0.{build} + +os: Visual Studio 2015 + +platform: x64 + + +matrix: + fast_finish: true + +# Should speed up repository cloning +# +shallow_clone: true +clone_depth: 5 + +#init: +# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + +#on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + +cache: + C:\Miniconda3-x64\pkgs -> appveyor.yml + +install: + - call scripts\\appveyor\\config.cmd + +build_script: + - call scripts\\appveyor\\build.cmd + +test_script: + - call scripts\\appveyor\\test.cmd + - diff --git a/README.rst b/README.rst index 5bac79b4..7203e881 100644 --- a/README.rst +++ b/README.rst @@ -50,6 +50,9 @@ sorts it by the ``X`` dimension: .. image:: https://travis-ci.org/PDAL/python.svg?branch=master :target: https://travis-ci.org/PDAL/python +.. image:: https://ci.appveyor.com/api/projects/status/of4kecyahpo8892d + :target: https://ci.appveyor.com/project/hobu/python/ + Requirements ================================================================================ diff --git a/scripts/appveyor/build.cmd b/scripts/appveyor/build.cmd new file mode 100644 index 00000000..8306aebb --- /dev/null +++ b/scripts/appveyor/build.cmd @@ -0,0 +1,7 @@ +call "%CONDA_ROOT%\Scripts\activate.bat" base +call conda install -c conda-forge -y pdal cython packaging + +call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 + +python setup.py build + diff --git a/scripts/appveyor/config.cmd b/scripts/appveyor/config.cmd new file mode 100644 index 00000000..3188a8bc --- /dev/null +++ b/scripts/appveyor/config.cmd @@ -0,0 +1,16 @@ +@echo off + +set "CONDA_ROOT=C:\Miniconda3-x64" +set PATH=%CONDA_ROOT%;%CONDA_ROOT%\\scripts;%CONDA_ROOT%\\Library\\bin;%PATH%;C:\\Program Files (x86)\\CMake\\bin +conda config --set always_yes yes +conda config --add channels conda-forge +conda config --set auto_update_conda no +conda config --set channel_priority true +conda update -q --all +conda info +python -c "import sys; print(sys.version)" +python -c "import sys; print(sys.executable)" +python -c "import sys; print(sys.prefix)" + +dir + diff --git a/scripts/appveyor/test.cmd b/scripts/appveyor/test.cmd new file mode 100644 index 00000000..61c3b221 --- /dev/null +++ b/scripts/appveyor/test.cmd @@ -0,0 +1,3 @@ +call "%CONDA_ROOT%\Scripts\activate.bat" base + +python setup.py test From 5cc72047a6d9fa3b6bc9e35cb0a1043cb7df10a1 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 29 Aug 2019 17:01:16 -0500 Subject: [PATCH 019/343] Fix ups for PDAL 2.0 (#30) * Checkpoint. * Fix tests. * Update for new name of MemoryViewReader. * clean up to catch up with PDAL 2.0 * add back logging test * shut off log test * bump version to 2.2.0 in preparation for release --- VERSION.txt | 2 +- pdal/PyArray.cpp | 339 +++++++++++++++++++++++++++++++++++++++++ pdal/PyArray.hpp | 231 ++++++---------------------- pdal/PyPipeline.cpp | 123 +++++++-------- pdal/PyPipeline.hpp | 19 +-- pdal/__init__.py | 2 +- pdal/libpdalpython.pyx | 29 ++-- setup.cfg | 3 +- setup.py | 6 +- test/data/test3d.npy | Bin 0 -> 368 bytes test/test_pipeline.py | 39 ++--- 11 files changed, 492 insertions(+), 301 deletions(-) create mode 100644 pdal/PyArray.cpp create mode 100644 test/data/test3d.npy diff --git a/VERSION.txt b/VERSION.txt index 9671f9a9..e3a4f193 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.1.7 \ No newline at end of file +2.2.0 \ No newline at end of file diff --git a/pdal/PyArray.cpp b/pdal/PyArray.cpp new file mode 100644 index 00000000..73e6410d --- /dev/null +++ b/pdal/PyArray.cpp @@ -0,0 +1,339 @@ +/****************************************************************************** +* Copyright (c) 2019, Hobu Inc. (info@hobu.co) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "PyArray.hpp" +#include + +#include + +namespace pdal +{ +namespace python +{ + +namespace +{ + +Dimension::Type pdalType(int t) +{ + using namespace Dimension; + + switch (t) + { + case NPY_FLOAT32: + return Type::Float; + case NPY_FLOAT64: + return Type::Double; + case NPY_INT8: + return Type::Signed8; + case NPY_INT16: + return Type::Signed16; + case NPY_INT32: + return Type::Signed32; + case NPY_INT64: + return Type::Signed64; + case NPY_UINT8: + return Type::Unsigned8; + case NPY_UINT16: + return Type::Unsigned16; + case NPY_UINT32: + return Type::Unsigned32; + case NPY_UINT64: + return Type::Unsigned64; + default: + return Type::None; + } + assert(0); + + return Type::None; +} + +std::string toString(PyObject *pname) +{ + PyObject* r = PyObject_Str(pname); + if (!r) + throw pdal_error("couldn't make string representation value"); + Py_ssize_t size; + return std::string(PyUnicode_AsUTF8AndSize(r, &size)); +} + +} // unnamed namespace + +Array::Array() : m_array(nullptr) +{ + if (_import_array() < 0) + throw pdal_error("Could not import numpy.core.multiarray."); +} + +Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true) +{ + if (_import_array() < 0) + throw pdal_error("Could not import numpy.core.multiarray."); + + Py_XINCREF(array); + + PyArray_Descr *dtype = PyArray_DTYPE(m_array); + npy_intp ndims = PyArray_NDIM(m_array); + npy_intp *shape = PyArray_SHAPE(m_array); + int numFields = (dtype->fields == Py_None) ? + 0 : + static_cast(PyDict_Size(dtype->fields)); + + int xyz = 0; + if (numFields == 0) + { + if (ndims != 3) + throw pdal_error("Array without fields must have 3 dimensions."); + m_fields.push_back({"Intensity", pdalType(dtype->type_num), 0}); + } + else + { + PyObject *names_dict = dtype->fields; + PyObject *names = PyDict_Keys(names_dict); + PyObject *values = PyDict_Values(names_dict); + if (!names || !values) + throw pdal_error("Bad field specification in numpy array."); + + for (int i = 0; i < numFields; ++i) + { + std::string name = toString(PyList_GetItem(names, i)); + if (name == "X") + xyz |= 1; + else if (name == "Y") + xyz |= 2; + else if (name == "Z") + xyz |= 4; + PyObject *tup = PyList_GetItem(values, i); + + // Get offset. + size_t offset = PyLong_AsLong(PySequence_Fast_GET_ITEM(tup, 1)); + + // Get type. + PyArray_Descr *descriptor = + (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0); + Dimension::Type type = pdalType(descriptor->type_num); + if (type == Dimension::Type::None) + throw pdal_error("Incompatible type for field '" + name + "'."); + + m_fields.push_back({name, type, offset}); + } + + if (xyz != 0 && xyz != 7) + throw pdal_error("Array fields must contain all or none " + "of X, Y and Z"); + if (xyz == 0 && ndims != 3) + throw pdal_error("Array without named X/Y/Z fields " + "must have three dimensions."); + } + if (xyz == 0) + m_shape = { (size_t)shape[0], (size_t)shape[1], (size_t)shape[2] }; + m_rowMajor = !(PyArray_FLAGS(m_array) & NPY_ARRAY_F_CONTIGUOUS); +} + +Array::~Array() +{ + if (m_array) + Py_XDECREF((PyObject *)m_array); +} + + +void Array::update(PointViewPtr view) +{ + if (m_array) + Py_XDECREF((PyObject *)m_array); + m_array = nullptr; // Just in case of an exception. + + Dimension::IdList dims = view->dims(); + npy_intp size = view->size(); + + PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view); + if (!dtype_dict) + throw pdal_error("Unable to build numpy dtype " + "description dictionary"); + + PyArray_Descr *dtype = nullptr; + if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL) + throw pdal_error("Unable to build numpy dtype"); + Py_XDECREF(dtype_dict); + + // This is a 1 x size array. + m_array = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype, + 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr); + + // copy the data + DimTypeList types = view->dimTypes(); + for (PointId idx = 0; idx < view->size(); idx++) + { + char *p = (char *)PyArray_GETPTR1(m_array, idx); + view->getPackedPoint(types, idx, p); + } +} + + +//ABELL - Who's responsible for incrementing the ref count? +PyArrayObject *Array::getPythonArray() const +{ + return m_array; +} + +PyObject* Array::buildNumpyDescription(PointViewPtr view) const +{ + // Build up a numpy dtype dictionary + // + // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', + // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], + // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', + // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine', + // 'Classification', 'ScanAngleRank', 'UserData', + // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']} + // + + Dimension::IdList dims = view->dims(); + + PyObject* dict = PyDict_New(); + PyObject* sizes = PyList_New(dims.size()); + PyObject* formats = PyList_New(dims.size()); + PyObject* titles = PyList_New(dims.size()); + + for (size_t i = 0; i < dims.size(); ++i) + { + Dimension::Id id = dims[i]; + Dimension::Type t = view->dimType(id); + npy_intp stride = view->dimSize(id); + + std::string name = view->dimName(id); + + std::string kind("i"); + Dimension::BaseType b = Dimension::base(t); + if (b == Dimension::BaseType::Unsigned) + kind = "u"; + else if (b == Dimension::BaseType::Signed) + kind = "i"; + else if (b == Dimension::BaseType::Floating) + kind = "f"; + else + throw pdal_error("Unable to map kind '" + kind + + "' to PDAL dimension type"); + + std::stringstream oss; + oss << kind << stride; + PyObject* pySize = PyLong_FromLong(stride); + PyObject* pyTitle = PyUnicode_FromString(name.c_str()); + PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str()); + + PyList_SetItem(sizes, i, pySize); + PyList_SetItem(titles, i, pyTitle); + PyList_SetItem(formats, i, pyFormat); + } + + PyDict_SetItemString(dict, "names", titles); + PyDict_SetItemString(dict, "formats", formats); + + return dict; +} + +bool Array::rowMajor() const +{ + return m_rowMajor; +} + +Array::Shape Array::shape() const +{ + return m_shape; +} + +const Array::Fields& Array::fields() const +{ + return m_fields; +} + +ArrayIter& Array::iterator() +{ + ArrayIter *it = new ArrayIter(*this); + m_iterators.push_back(std::unique_ptr(it)); + return *it; +} + +ArrayIter::ArrayIter(Array& array) +{ + m_iter = NpyIter_New(array.getPythonArray(), + NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK, + NPY_KEEPORDER, NPY_NO_CASTING, NULL); + if (!m_iter) + throw pdal_error("Unable to create numpy iterator."); + + char *itererr; + m_iterNext = NpyIter_GetIterNext(m_iter, &itererr); + if (!m_iterNext) + { + NpyIter_Deallocate(m_iter); + throw pdal_error(std::string("Unable to create numpy iterator: ") + + itererr); + } + m_data = NpyIter_GetDataPtrArray(m_iter); + m_stride = NpyIter_GetInnerStrideArray(m_iter); + m_size = NpyIter_GetInnerLoopSizePtr(m_iter); + m_done = false; +} + +ArrayIter::~ArrayIter() +{ + NpyIter_Deallocate(m_iter); +} + +ArrayIter& ArrayIter::operator++() +{ + if (m_done) + return *this; + + if (--(*m_size)) + *m_data += *m_stride; + else if (!m_iterNext(m_iter)) + m_done = true; + return *this; +} + +ArrayIter::operator bool () const +{ + return !m_done; +} + +char * ArrayIter::operator * () const +{ + return *m_data; +} + +} // namespace python +} // namespace pdal + diff --git a/pdal/PyArray.hpp b/pdal/PyArray.hpp index 8e075d31..e22e20b5 100644 --- a/pdal/PyArray.hpp +++ b/pdal/PyArray.hpp @@ -1,5 +1,5 @@ /****************************************************************************** -* Copyright (c) 2011, Michael P. Gerlek (mpg@flaxen.com) +* Copyright (c) 2019, Hobu Inc. (info@hobu.co) * * All rights reserved. * @@ -13,7 +13,7 @@ * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided * with the distribution. -* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* * Neither the name of Hobu, Inc. nor the * names of its contributors may be used to endorse or promote * products derived from this software without specific prior * written permission. @@ -34,204 +34,69 @@ #pragma once -#include - -#include - -#pragma warning(disable: 4127) // conditional expression is constant - - -#include -#undef toupper -#undef tolower -#undef isspace +#include -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include - -// forward declare PyObject so we don't need the python headers everywhere -// see: http://mail.python.org/pipermail/python-dev/2003-August/037601.html -#ifndef PyObject_HEAD -struct _object; -typedef _object PyObject; -#endif +#include +#include namespace pdal { namespace python { +class ArrayIter; class PDAL_DLL Array { public: + using Shape = std::array; + using Fields = std::vector; + + // Create an array for reading data from PDAL. + Array(); + + // Create an array for writing data to PDAL. + Array(PyArrayObject* array); - Array() : m_py_array(0), m_own_array(true) - { -#undef NUMPY_IMPORT_ARRAY_RETVAL -#define NUMPY_IMPORT_ARRAY_RETVAL - import_array(); - } - - Array(PyObject* array) : m_py_array(array), m_own_array(false) - { -#undef NUMPY_IMPORT_ARRAY_RETVAL -#define NUMPY_IMPORT_ARRAY_RETVAL - import_array(); - if (!PyArray_Check(array)) - throw pdal::pdal_error("pdal::python::Array constructor object is not a numpy array"); - Py_XINCREF(array); - - } - - ~Array() - { - cleanup(); - } - - - inline void update(PointViewPtr view) - { - typedef std::unique_ptr> DataPtr; - cleanup(); - int nd = 1; - Dimension::IdList dims = view->dims(); - npy_intp mydims = view->size(); - npy_intp* ndims = &mydims; - std::vector strides(dims.size()); - - DataPtr pdata( new std::vector(view->pointSize()* view->size(), 0)); - - PyArray_Descr *dtype = nullptr; - PyObject * dtype_dict = (PyObject*)buildNumpyDescription(view); - if (!dtype_dict) - throw pdal_error("Unable to build numpy dtype description dictionary"); - - int did_convert = PyArray_DescrConverter(dtype_dict, &dtype); - if (did_convert == NPY_FAIL) - throw pdal_error("Unable to build numpy dtype"); - Py_XDECREF(dtype_dict); - -#ifdef NPY_ARRAY_CARRAY - int flags = NPY_ARRAY_CARRAY; -#else - int flags = NPY_CARRAY; -#endif - uint8_t* sp = pdata.get()->data(); - PyObject * pyArray = PyArray_NewFromDescr(&PyArray_Type, - dtype, - nd, - ndims, - 0, - sp, - flags, - NULL); - - // copy the data - uint8_t* p(sp); - DimTypeList types = view->dimTypes(); - for (PointId idx = 0; idx < view->size(); idx++) - { - p = sp + (view->pointSize() * idx); - view->getPackedPoint(types, idx, (char*)p); - } - - m_py_array = pyArray; - m_data_array = std::move(pdata); - } - - - inline PyObject* getPythonArray() const - { - return m_py_array; - } + ~Array(); + void update(PointViewPtr view); + PyArrayObject *getPythonArray() const; + bool rowMajor() const; + Shape shape() const; + const Fields& fields() const; + ArrayIter& iterator(); private: + inline PyObject* buildNumpyDescription(PointViewPtr view) const; - inline void cleanup() - { - PyObject* p = (PyObject*)(m_py_array); - if (m_own_array) - { - m_data_array.reset(); - } - - Py_XDECREF(p); - } - - inline PyObject* buildNumpyDescription(PointViewPtr view) const - { - - // Build up a numpy dtype dictionary - // - // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'], - // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', 'NumberOfReturns', - // 'ScanDirectionFlag', 'EdgeOfFlightLine', 'Classification', - // 'ScanAngleRank', 'UserData', 'PointSourceId', 'GpsTime', 'Red', 'Green', - // 'Blue']} - // - - std::stringstream oss; - Dimension::IdList dims = view->dims(); - - PyObject* dict = PyDict_New(); - PyObject* sizes = PyList_New(dims.size()); - PyObject* formats = PyList_New(dims.size()); - PyObject* titles = PyList_New(dims.size()); - - for (Dimension::IdList::size_type i=0; i < dims.size(); ++i) - { - Dimension::Id id = (dims[i]); - Dimension::Type t = view->dimType(id); - npy_intp stride = view->dimSize(id); - - std::string name = view->dimName(id); - - std::string kind("i"); - Dimension::BaseType b = Dimension::base(t); - if (b == Dimension::BaseType::Unsigned) - kind = "u"; - else if (b == Dimension::BaseType::Signed) - kind = "i"; - else if (b == Dimension::BaseType::Floating) - kind = "f"; - else - { - std::stringstream o; - oss << "unable to map kind '" << kind <<"' to PDAL dimension type"; - throw pdal::pdal_error(o.str()); - } - - oss << kind << stride; - PyObject* pySize = PyLong_FromLong(stride); - PyObject* pyTitle = PyUnicode_FromString(name.c_str()); - PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str()); - - PyList_SetItem(sizes, i, pySize); - PyList_SetItem(titles, i, pyTitle); - PyList_SetItem(formats, i, pyFormat); - - oss.str(""); - } - - PyDict_SetItemString(dict, "names", titles); - PyDict_SetItemString(dict, "formats", formats); - - // PyObject* obj = PyUnicode_AsASCIIString(PyObject_Str(dict)); - // const char* s = PyBytes_AsString(obj); - // std::string output(s); - // std::cout << "array: " << output << std::endl; - return dict; - } - - - - - PyObject* m_py_array; - std::unique_ptr > m_data_array; - bool m_own_array; + PyArrayObject* m_array; Array& operator=(Array const& rhs); + Fields m_fields; + bool m_rowMajor; + Shape m_shape; + std::vector> m_iterators; +}; + +class ArrayIter +{ +public: + ArrayIter(const ArrayIter&) = delete; + + ArrayIter(Array& array); + ~ArrayIter(); + + ArrayIter& operator++(); + operator bool () const; + char *operator * () const; + +private: + NpyIter *m_iter; + NpyIter_IterNextFunc *m_iterNext; + char **m_data; + npy_intp *m_size; + npy_intp *m_stride; + bool m_done; }; } // namespace python diff --git a/pdal/PyPipeline.cpp b/pdal/PyPipeline.cpp index 27c377f9..66c613e3 100644 --- a/pdal/PyPipeline.cpp +++ b/pdal/PyPipeline.cpp @@ -33,124 +33,125 @@ ****************************************************************************/ #include "PyPipeline.hpp" -#ifdef PDAL_HAVE_LIBXML2 -#include -#endif #ifndef _WIN32 #include #endif #include -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include -#include "PyArray.hpp" #include #include -#include -#include - -namespace libpdalpython -{ -using namespace pdal::python; +#include "PyArray.hpp" -Pipeline::Pipeline(std::string const& json, std::vector arrays) +namespace pdal +{ +namespace python { +// Create a pipeline for writing data to PDAL +Pipeline::Pipeline(std::string const& json, std::vector arrays) : + m_executor(new PipelineExecutor(json)) +{ #ifndef _WIN32 + // See comment in alternate constructor below. ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); - ::dlopen("libpdal_plugin_reader_numpy.so", RTLD_NOLOAD | RTLD_GLOBAL); #endif -#undef NUMPY_IMPORT_ARRAY_RETVAL -#define NUMPY_IMPORT_ARRAY_RETVAL - import_array(); - - m_executor = std::shared_ptr(new pdal::PipelineExecutor(json)); + if (_import_array() < 0) + throw pdal_error("Could not impory numpy.core.multiarray."); - pdal::PipelineManager& manager = m_executor->getManager(); + PipelineManager& manager = m_executor->getManager(); std::stringstream strm(json); manager.readPipeline(strm); + std::vector roots = manager.roots(); + if (roots.size() != 1) + throw pdal_error("Filter pipeline must contain a single root stage."); - pdal::Stage *r = manager.getStage(); - if (!r) - throw pdal::pdal_error("pipeline had no stages!"); - -#if PDAL_VERSION_MAJOR > 1 || PDAL_VERSION_MINOR >=8 - int counter = 1; - for (auto array: arrays) + for (auto array : arrays) { // Create numpy reader for each array - pdal::Options options; - std::stringstream tag; - tag << "readers_numpy" << counter; - pdal::StageCreationOptions opts { "", "readers.numpy", nullptr, options, tag.str()}; - pdal::Stage& reader = manager.makeReader(opts); - - pdal::NumpyReader* np_reader = dynamic_cast(&reader); - if (!np_reader) - throw pdal::pdal_error("couldn't cast reader!"); - + // Options + + Options options; + options.add("order", array->rowMajor() ? + MemoryViewReader::Order::RowMajor : + MemoryViewReader::Order::ColumnMajor); + options.add("shape", MemoryViewReader::Shape(array->shape())); + + Stage& s = manager.makeReader("", "readers.memoryview", options); + MemoryViewReader& r = dynamic_cast(s); + for (auto f : array->fields()) + r.pushField(f); + + ArrayIter& iter = array->iterator(); + auto incrementer = [&iter](PointId id) -> char * + { + if (! iter) + return nullptr; + + char *c = *iter; + ++iter; + return c; + }; + + r.setIncrementer(incrementer); PyObject* parray = (PyObject*)array->getPythonArray(); if (!parray) - throw pdal::pdal_error("array was none!"); - - np_reader->setArray(parray); - - r->setInput(reader); - counter++; + throw pdal_error("array was none!"); + roots[0]->setInput(r); } -#endif manager.validateStageOptions(); } -Pipeline::Pipeline(std::string const& json) +// Create a pipeline for reading data from PDAL +Pipeline::Pipeline(std::string const& json) : + m_executor(new PipelineExecutor(json)) { // Make the symbols in pdal_base global so that they're accessible // to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL, // which means that without this, symbols in libpdal_base aren't available // for resolution of symbols on future runtime linking. This is an issue - // on Apline and other Linux variants that doesn't use UNIQUE symbols - // for C++ template statics. only + // on Alpine and other Linux variants that don't use UNIQUE symbols + // for C++ template statics only. Without this, you end up with multiple + // copies of template statics. #ifndef _WIN32 ::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL); #endif -#undef NUMPY_IMPORT_ARRAY_RETVAL -#define NUMPY_IMPORT_ARRAY_RETVAL - import_array(); - - m_executor = std::shared_ptr(new pdal::PipelineExecutor(json)); + if (_import_array() < 0) + throw pdal_error("Could not impory numpy.core.multiarray."); } Pipeline::~Pipeline() -{ -} +{} + void Pipeline::setLogLevel(int level) { m_executor->setLogLevel(level); } + int Pipeline::getLogLevel() const { return static_cast(m_executor->getLogLevel()); } + int64_t Pipeline::execute() { - - int64_t count = m_executor->execute(); - return count; + return m_executor->execute(); } bool Pipeline::validate() { - return m_executor->validate(); + auto res = m_executor->validate(); + return res; } std::vector Pipeline::getArrays() const @@ -160,16 +161,18 @@ std::vector Pipeline::getArrays() const if (!m_executor->executed()) throw python_error("call execute() before fetching arrays"); - const pdal::PointViewSet& pvset = m_executor->getManagerConst().views(); + const PointViewSet& pvset = m_executor->getManagerConst().views(); for (auto i: pvset) { //ABELL - Leak? - Array *array = new pdal::python::Array; + Array *array = new python::Array; array->update(i); output.push_back(array); } return output; } -} //namespace libpdalpython + +} // namespace python +} // namespace pdal diff --git a/pdal/PyPipeline.hpp b/pdal/PyPipeline.hpp index f12953bc..a1ddc30b 100644 --- a/pdal/PyPipeline.hpp +++ b/pdal/PyPipeline.hpp @@ -43,20 +43,12 @@ #include #include -#undef toupper -#undef tolower -#undef isspace - namespace pdal { namespace python { - class Array; -} -} -namespace libpdalpython -{ +class Array; class python_error : public std::runtime_error { @@ -65,10 +57,12 @@ class python_error : public std::runtime_error {} }; -class Pipeline { +class Pipeline +{ public: Pipeline(std::string const& json); - Pipeline(std::string const& json, std::vector arrays); + Pipeline(std::string const& json, + std::vector arrays); ~Pipeline(); int64_t execute(); @@ -98,4 +92,5 @@ class Pipeline { std::shared_ptr m_executor; }; -} +} // namespace python +} // namespace pdal diff --git a/pdal/__init__.py b/pdal/__init__.py index c287bae6..22246f3e 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.1.8' +__version__='2.2.0' from .pipeline import Pipeline from .array import Array diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index a33f182c..a2f215da 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -23,7 +23,6 @@ cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config": def getVersionString(): return versionString() - def getVersionMajor(): return versionMajor() def getVersionMinor(): @@ -39,10 +38,10 @@ def getPluginInstallPath(): cdef extern from "PyArray.hpp" namespace "pdal::python": cdef cppclass Array: - Array(object) except + - void* getPythonArray() except+ + Array(np.ndarray) except + + void *getPythonArray() except+ -cdef extern from "PyPipeline.hpp" namespace "libpdalpython": +cdef extern from "PyPipeline.hpp" namespace "pdal::python": cdef cppclass Pipeline: Pipeline(const char* ) except + Pipeline(const char*, vector[Array*]& ) except + @@ -56,11 +55,9 @@ cdef extern from "PyPipeline.hpp" namespace "libpdalpython": int getLogLevel() void setLogLevel(int) - - cdef class PyArray: cdef Array *thisptr - def __cinit__(self, object array): + def __cinit__(self, np.ndarray array): self.thisptr = new Array(array) def __dealloc__(self): del self.thisptr @@ -109,24 +106,14 @@ cdef class PyPipeline: cdef Array* a if arrays is not None: + print("Looping arrays\n") for array in arrays: a = new Array(array) c_arrays.push_back(a) - if PY_MAJOR_VERSION >= 3: - if arrays: - self.thisptr = new Pipeline(json.encode('UTF-8'), c_arrays) - else: - self.thisptr = new Pipeline(json.encode('UTF-8')) + self.thisptr = new Pipeline(json.encode('UTF-8'), c_arrays) else: - if arrays: - self.thisptr = new Pipeline(json, c_arrays) - else: - self.thisptr = new Pipeline(json) -# if arrays: -# self.thisptr = new Pipeline(json.encode('UTF-8'), c_arrays) -# else: -# self.thisptr = new Pipeline(json.encode('UTF-8')) + self.thisptr = new Pipeline(json.encode('UTF-8')) def __dealloc__(self): del self.thisptr @@ -158,6 +145,7 @@ cdef class PyPipeline: return json.loads(j) property arrays: + def __get__(self): v = self.thisptr.getArrays() output = [] @@ -171,6 +159,7 @@ cdef class PyPipeline: inc(it) return output + def execute(self): if not self.thisptr: raise Exception("C++ Pipeline object not constructed!") diff --git a/setup.cfg b/setup.cfg index 861a9f55..7f75bb2c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,4 @@ [egg_info] -tag_build = +tag_build = tag_date = 0 tag_svn_revision = 0 - diff --git a/setup.py b/setup.py index 8bd2fd3e..84d7fdf3 100644 --- a/setup.py +++ b/setup.py @@ -182,7 +182,7 @@ def get_pdal_config(option): -sources=['pdal/libpdalpython'+ext, "pdal/PyPipeline.cpp" ] +sources=['pdal/libpdalpython'+ext, "pdal/PyPipeline.cpp", "pdal/PyArray.cpp" ] extensions = [DistutilsExtension("*", sources, include_dirs=include_dirs, @@ -192,12 +192,12 @@ def get_pdal_config(option): extra_link_args=extra_link_args,)] if USE_CYTHON and "clean" not in sys.argv: from Cython.Build import cythonize - extensions= cythonize(extensions, language="c++") + extensions= cythonize(extensions, compiler_directives={'language_level':3}) setup_args = dict( name = 'PDAL', version = str(module_version), - requires = ['Python (>=2.7)', 'Numpy'], + requires = ['Python (>=3.0)', 'Numpy'], description = 'Point cloud data processing', license = 'BSD', keywords = 'point cloud spatial', diff --git a/test/data/test3d.npy b/test/data/test3d.npy new file mode 100644 index 0000000000000000000000000000000000000000..58347b5134d7ccb0a85a3b0d641a3be1c6c76c64 GIT binary patch literal 368 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+i=qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I%IIts=*3Pzeb3bhL40WJm<@WCF!RdImOYEaq)N}EFINC@o!r5mCl`~y&$ zAr2}Jr5Q3Id<7`&kOkoTK2QYVUx3mH#Snf0lx8S_@CBeWLnV|Cr58Zy O4N%&l3L+m+ Date: Fri, 6 Sep 2019 15:09:58 -0500 Subject: [PATCH 020/343] prepare 2.2.0 --- setup.py | 8 +++----- test/test_pipeline.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index 84d7fdf3..e19e6dd2 100644 --- a/setup.py +++ b/setup.py @@ -156,9 +156,9 @@ def get_pdal_config(option): if os.name != 'nt': extra_compile_args += ['-g','-O0'] -# readers.numpy doesn't exist until PDAL 1.8 -if PDALVERSION is not None and PDALVERSION >= Version('1.8'): - libraries.append('pdal_plugin_reader_numpy') +if PDALVERSION < Version('2.0.0'): + raise Exception("PDAL version '%s' is not compatible with PDAL Python library version '%s'"%(PDALVERSION, module_version)) + if os.name in ['nt']: if os.environ.get('OSGEO4W_ROOT'): @@ -168,8 +168,6 @@ def get_pdal_config(option): library_dirs = ['%s\Library\lib' % prefix] libraries = ['pdalcpp','pdal_util','ws2_32'] - if PDALVERSION >= Version('1.8'): - libraries.append('libpdal_plugin_reader_numpy') extra_compile_args = ['/DNOMINMAX',] diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 149c4008..949c8661 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -31,30 +31,30 @@ def fetch_json(self, filename): return output class TestPipeline(PDALTest): -# + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), "missing test data") def test_construction(self): """Can we construct a PDAL pipeline""" json = self.fetch_json('sort.json') r = pdal.Pipeline(json) -# + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), "missing test data") def test_execution(self): """Can we execute a PDAL pipeline""" x = self.fetch_json('sort.json') r = pdal.Pipeline(x) -# r.validate() + r.validate() r.execute() self.assertGreater(len(r.pipeline), 200) -# + def test_validate(self): """Do we complain with bad pipelines""" r = pdal.Pipeline(bad_json) with self.assertRaises(RuntimeError): r.validate() -# + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), "missing test data") def test_array(self): @@ -69,7 +69,7 @@ def test_array(self): a = arrays[0] self.assertAlmostEqual(a[0][0], 635619.85, 7) self.assertAlmostEqual(a[1064][2], 456.92, 7) -# + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), "missing test data") def test_metadata(self): @@ -82,8 +82,8 @@ def test_metadata(self): import json j = json.loads(metadata) self.assertEqual(j["metadata"]["readers.las"][0]["count"], 1065) -# -# + + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')), "missing test data") def test_no_execute(self): @@ -114,7 +114,7 @@ def test_schema(self): r.validate() r.execute() self.assertEqual(r.schema['schema']['dimensions'][0]['name'], 'X') -# + @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'chip.json')), "missing test data") def test_merged_arrays(self): @@ -125,7 +125,7 @@ def test_merged_arrays(self): r.execute() arrays = r.arrays self.assertEqual(len(arrays), 43) -# + class TestArrayLoad(PDALTest): From 2bad46e6bcb6512ea41bf41915a66040ea94c660 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 6 Sep 2019 15:24:37 -0500 Subject: [PATCH 021/343] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..7d405f9e --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,36 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +pool: + vmImage: 'ubuntu-latest' +strategy: + matrix: + Python27: + python.version: '2.7' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- script: | + pip install pytest pytest-azurepipelines + pytest + displayName: 'pytest' From dfd1d96a61d36de436a6aeb7aea395c362bcab19 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 6 Sep 2019 15:35:35 -0500 Subject: [PATCH 022/343] AzP setup --- azure-pipelines.yml | 39 ++++----------------------------- scripts/azp/linux.yml | 21 ++++++++++++++++++ scripts/azp/osx.yml | 51 +++++++++++++++++++++++++++++++++++++++++++ scripts/azp/win.yml | 40 +++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 35 deletions(-) create mode 100644 scripts/azp/linux.yml create mode 100644 scripts/azp/osx.yml create mode 100644 scripts/azp/win.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7d405f9e..7677b24b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,36 +1,5 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python +jobs: + - template: ./scripts/azp/linux.yml + - template: ./scripts/azp/win.yml + - template: ./scripts/azp/osx.yml -trigger: -- master - -pool: - vmImage: 'ubuntu-latest' -strategy: - matrix: - Python27: - python.version: '2.7' - Python35: - python.version: '3.5' - Python36: - python.version: '3.6' - Python37: - python.version: '3.7' - -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' - -- script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install dependencies' - -- script: | - pip install pytest pytest-azurepipelines - pytest - displayName: 'pytest' diff --git a/scripts/azp/linux.yml b/scripts/azp/linux.yml new file mode 100644 index 00000000..a41017e1 --- /dev/null +++ b/scripts/azp/linux.yml @@ -0,0 +1,21 @@ +# -*- mode: yaml -*- + +jobs: +- job: linux + pool: + vmImage: ubuntu-16.04 + container: + image: pdal/pdal:latest + options: --privileged + timeoutInMinutes: 60 + steps: + - script: | + displayName: 'Dependencies' + apt-get update -y; apt-get install python3-pip -y + pip3 install numpy packaging cython + - script: | + python3 setup.py build + displayName: 'Build' + - script: | + python3 setup.py test + displayName: 'Test' diff --git a/scripts/azp/osx.yml b/scripts/azp/osx.yml new file mode 100644 index 00000000..80d23cf8 --- /dev/null +++ b/scripts/azp/osx.yml @@ -0,0 +1,51 @@ +# -*- mode: yaml -*- + +jobs: +- job: osx + pool: + vmImage: macOS-10.13 + timeoutInMinutes: 360 + steps: + - script: | + echo "Removing homebrew from Azure to avoid conflicts." + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > ~/uninstall_homebrew + chmod +x ~/uninstall_homebrew + ~/uninstall_homebrew -fq + rm ~/uninstall_homebrew + displayName: Remove homebrew + - bash: | + echo "##vso[task.prependpath]$CONDA/bin" + sudo chown -R $USER $CONDA + displayName: Add conda to PATH + + - script: | + ECHO ON + source activate base + conda create --yes --quiet --name pdal + displayName: Create conda environment + + - script: | + ECHO ON + source activate pdal + conda config --set always_yes True --set show_channel_urls True + conda install --yes --quiet --name pdal -c conda-forge cython packaging pdal conda-forge-ci-setup=2 -y + displayName: Install PDAL + - script: | + source activate pdal + export CI=azure + source run_conda_forge_build_setup + env: { + OSX_FORCE_SDK_DOWNLOAD: "1" + } + displayName: Configure OSX SDK + - script: | + source activate pdal + pip install numpy packaging cython + displayName: 'Dependencies' + - script: | + python setup.py build + displayName: 'Build' + - script: | + python setup.py test + displayName: 'Test' + diff --git a/scripts/azp/win.yml b/scripts/azp/win.yml new file mode 100644 index 00000000..4abdc798 --- /dev/null +++ b/scripts/azp/win.yml @@ -0,0 +1,40 @@ + +jobs: +- job: win + + pool: + vmImage: vs2017-win2016 + timeoutInMinutes: 360 + steps: + - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + displayName: Add conda to PATH + + - script: | + ECHO ON + call conda create --yes --quiet --name pdal + displayName: Create conda environment + + - script: | + ECHO ON + call activate pdal + call conda config --set always_yes True --set show_channel_urls True + call conda install --yes --quiet --name pdal -c conda-forge pdal numpy packaging cython -y + displayName: Install PDAL + - script: | + ECHO ON + call activate pdal + call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 + echo %PATH% + set CC=cl.exe + set CXX=cl.exe + python setup.py build + displayName: 'Build' + - script: | + ECHO ON + call activate pdal + call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 + set PYTHONHOME=%CONDA_PREFIX% + python setup.py test + displayName: 'Test' + + From c3dc657c94d33b17e266427cfd0254e73e5f5ee5 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 6 Sep 2019 15:40:29 -0500 Subject: [PATCH 023/343] azp indenting --- scripts/azp/linux.yml | 6 +++--- scripts/azp/osx.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/azp/linux.yml b/scripts/azp/linux.yml index a41017e1..23ed0545 100644 --- a/scripts/azp/linux.yml +++ b/scripts/azp/linux.yml @@ -10,11 +10,11 @@ jobs: timeoutInMinutes: 60 steps: - script: | + apt-get update -y; apt-get install python3-pip -y + pip3 install numpy packaging cython displayName: 'Dependencies' - apt-get update -y; apt-get install python3-pip -y - pip3 install numpy packaging cython - script: | - python3 setup.py build + python3 setup.py build displayName: 'Build' - script: | python3 setup.py test diff --git a/scripts/azp/osx.yml b/scripts/azp/osx.yml index 80d23cf8..4dc46dfb 100644 --- a/scripts/azp/osx.yml +++ b/scripts/azp/osx.yml @@ -34,9 +34,9 @@ jobs: source activate pdal export CI=azure source run_conda_forge_build_setup - env: { - OSX_FORCE_SDK_DOWNLOAD: "1" - } + env: { + OSX_FORCE_SDK_DOWNLOAD: "1" + } displayName: Configure OSX SDK - script: | source activate pdal From 31e8b01c58349f7e3763f025302cb46cf6a98a2a Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 6 Sep 2019 15:48:32 -0500 Subject: [PATCH 024/343] more azp churn --- scripts/azp/linux.yml | 2 +- scripts/azp/osx.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/azp/linux.yml b/scripts/azp/linux.yml index 23ed0545..d5a98b99 100644 --- a/scripts/azp/linux.yml +++ b/scripts/azp/linux.yml @@ -10,7 +10,7 @@ jobs: timeoutInMinutes: 60 steps: - script: | - apt-get update -y; apt-get install python3-pip -y + sudo apt-get update -y; sudo apt-get install python3-pip -y pip3 install numpy packaging cython displayName: 'Dependencies' - script: | diff --git a/scripts/azp/osx.yml b/scripts/azp/osx.yml index 4dc46dfb..80d23cf8 100644 --- a/scripts/azp/osx.yml +++ b/scripts/azp/osx.yml @@ -34,9 +34,9 @@ jobs: source activate pdal export CI=azure source run_conda_forge_build_setup - env: { - OSX_FORCE_SDK_DOWNLOAD: "1" - } + env: { + OSX_FORCE_SDK_DOWNLOAD: "1" + } displayName: Configure OSX SDK - script: | source activate pdal From bc4f4b509fadb30ce1aa665a0728793cca70fa18 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 6 Sep 2019 16:13:35 -0500 Subject: [PATCH 025/343] azp linux/osx churn --- scripts/azp/linux.yml | 2 +- scripts/azp/osx.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/azp/linux.yml b/scripts/azp/linux.yml index d5a98b99..91ba86ac 100644 --- a/scripts/azp/linux.yml +++ b/scripts/azp/linux.yml @@ -10,7 +10,7 @@ jobs: timeoutInMinutes: 60 steps: - script: | - sudo apt-get update -y; sudo apt-get install python3-pip -y + sudo apt-get install python3-pip -y pip3 install numpy packaging cython displayName: 'Dependencies' - script: | diff --git a/scripts/azp/osx.yml b/scripts/azp/osx.yml index 80d23cf8..60d1c516 100644 --- a/scripts/azp/osx.yml +++ b/scripts/azp/osx.yml @@ -28,7 +28,7 @@ jobs: ECHO ON source activate pdal conda config --set always_yes True --set show_channel_urls True - conda install --yes --quiet --name pdal -c conda-forge cython packaging pdal conda-forge-ci-setup=2 -y + conda install --yes --quiet --name pdal -c conda-forge cython packaging pdal numpy conda-forge-ci-setup=2 -y displayName: Install PDAL - script: | source activate pdal From 9c7c5d228513fa68b58ac6ef9412d67fdea1fba9 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 6 Sep 2019 16:16:17 -0500 Subject: [PATCH 026/343] apt-get update --- scripts/azp/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/azp/linux.yml b/scripts/azp/linux.yml index 91ba86ac..d5a98b99 100644 --- a/scripts/azp/linux.yml +++ b/scripts/azp/linux.yml @@ -10,7 +10,7 @@ jobs: timeoutInMinutes: 60 steps: - script: | - sudo apt-get install python3-pip -y + sudo apt-get update -y; sudo apt-get install python3-pip -y pip3 install numpy packaging cython displayName: 'Dependencies' - script: | From 2056460e7baf257c5ecd67907cc5fa2c6270c635 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Fri, 6 Sep 2019 16:27:04 -0500 Subject: [PATCH 027/343] osx activate pdal env --- scripts/azp/osx.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/azp/osx.yml b/scripts/azp/osx.yml index 60d1c516..1598d2ab 100644 --- a/scripts/azp/osx.yml +++ b/scripts/azp/osx.yml @@ -43,9 +43,11 @@ jobs: pip install numpy packaging cython displayName: 'Dependencies' - script: | + source activate pdal python setup.py build displayName: 'Build' - script: | + source activate pdal python setup.py test displayName: 'Test' From 66870b6ff39ca268ea6c2410cba1c8e188f58bfe Mon Sep 17 00:00:00 2001 From: Bas Couwenberg Date: Sat, 7 Sep 2019 15:35:22 +0200 Subject: [PATCH 028/343] Fix clean target. (#32) `PDALVERSION` is not set when executing `setup.py clean`. Ref: #24 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e19e6dd2..7b56accc 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def get_pdal_config(option): if os.name != 'nt': extra_compile_args += ['-g','-O0'] -if PDALVERSION < Version('2.0.0'): +if PDALVERSION is not None and PDALVERSION < Version('2.0.0'): raise Exception("PDAL version '%s' is not compatible with PDAL Python library version '%s'"%(PDALVERSION, module_version)) From 588c38fe78d635acbd232d4d22216281d54d1804 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Sat, 7 Sep 2019 08:36:00 -0500 Subject: [PATCH 029/343] bump version to 2.2.1 to fix 'clean' target --- pdal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/__init__.py b/pdal/__init__.py index 22246f3e..e13f131c 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.2.0' +__version__='2.2.1' from .pipeline import Pipeline from .array import Array From 648abf76d7d8b8670e7d7d920b627b45a52a7fef Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Thu, 19 Sep 2019 06:16:03 +0200 Subject: [PATCH 030/343] Make sure Shape is initialized. (#36) --- VERSION.txt | 2 +- pdal/PyArray.hpp | 2 +- pdal/libpdalpython.pyx | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index e3a4f193..fae692e4 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -2.2.0 \ No newline at end of file +2.2.1 \ No newline at end of file diff --git a/pdal/PyArray.hpp b/pdal/PyArray.hpp index e22e20b5..d77b3d10 100644 --- a/pdal/PyArray.hpp +++ b/pdal/PyArray.hpp @@ -74,7 +74,7 @@ class PDAL_DLL Array Array& operator=(Array const& rhs); Fields m_fields; bool m_rowMajor; - Shape m_shape; + Shape m_shape {}; std::vector> m_iterators; }; diff --git a/pdal/libpdalpython.pyx b/pdal/libpdalpython.pyx index a2f215da..690b0de1 100644 --- a/pdal/libpdalpython.pyx +++ b/pdal/libpdalpython.pyx @@ -106,7 +106,6 @@ cdef class PyPipeline: cdef Array* a if arrays is not None: - print("Looping arrays\n") for array in arrays: a = new Array(array) c_arrays.push_back(a) From 0da1fb7dcba0e006cda4700048dac8dfba556548 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 19 Sep 2019 16:06:41 -0500 Subject: [PATCH 031/343] remove VERSION.txt, prepare 2.2.2 release --- MANIFEST.in | 2 +- VERSION.txt | 1 - pdal/__init__.py | 2 +- setup.py | 3 --- test/test_pipeline.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 35 insertions(+), 6 deletions(-) delete mode 100644 VERSION.txt diff --git a/MANIFEST.in b/MANIFEST.in index ff6f5924..57a62d95 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ exclude *.txt exclude MANIFEST.in -include CHANGES.txt README.rst VERSION.txt +include CHANGES.txt README.rst recursive-include test *.py recursive-include pdal *.pyx recursive-include pdal *.hpp diff --git a/VERSION.txt b/VERSION.txt deleted file mode 100644 index fae692e4..00000000 --- a/VERSION.txt +++ /dev/null @@ -1 +0,0 @@ -2.2.1 \ No newline at end of file diff --git a/pdal/__init__.py b/pdal/__init__.py index e13f131c..4d775cb6 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.2.1' +__version__='2.2.2' from .pipeline import Pipeline from .array import Array diff --git a/setup.py b/setup.py index 7b56accc..5ab97be4 100644 --- a/setup.py +++ b/setup.py @@ -91,9 +91,6 @@ def get_pdal_config(option): if sys.version_info >= (3,): open_kwds['encoding'] = 'utf-8' -with open('VERSION.txt', 'w', **open_kwds) as fp: - fp.write(str(module_version)) - with open('README.rst', 'r', **open_kwds) as fp: readme = fp.read() diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 949c8661..823ae1ff 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -159,6 +159,39 @@ def test_merged_arrays(self): self.assertEqual(len(data), 12) self.assertEqual(data['Intensity'].sum(), 1926) + def test_read_arrays(self): + """Can we read and filter data from a list of arrays to PDAL""" + if Version(pdal.info.version) < Version('1.8'): + return True + + # just some dummy data + x_vals = [1.0, 2.0, 3.0, 4.0, 5.0] + y_vals = [6.0, 7.0, 8.0, 9.0, 10.0] + z_vals = [1.5, 3.5, 5.5, 7.5, 9.5] + test_data = np.array( + [(x, y, z) for x, y, z in zip(x_vals, y_vals, z_vals)], + dtype=[('X', np.float), ('Y', np.float), ('Z', np.float)] + ) + + pipeline = """ + { + "pipeline": [ + { + "type":"filters.range", + "limits":"X[2.5:4.5]" + } + ] + } + """ + + p = pdal.Pipeline(pipeline, arrays=[test_data,]) + p.loglevel = 8 + count = p.execute() + arrays = p.arrays + self.assertEqual(count, 2) + self.assertEqual(len(arrays), 1) + + class TestDimensions(PDALTest): def test_fetch_dimensions(self): """Ask PDAL for its valid dimensions list""" From 3eea75bcfaeffa85d9e0c91e05f241e3760448db Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 12:47:07 -0600 Subject: [PATCH 032/343] pio module for constructing and executing pdal pipelines in a pythonic way and returning results --- pdal/pio.py | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 pdal/pio.py diff --git a/pdal/pio.py b/pdal/pio.py new file mode 100644 index 00000000..fd1334f7 --- /dev/null +++ b/pdal/pio.py @@ -0,0 +1,111 @@ +import json +from functools import partial +from collections import defaultdict +from itertools import chain +import copy +import warnings + +import pdal + +DEFAULT_STAGE_PARAMS = defaultdict(dict) +DEFAULT_STAGE_PARAMS.update({ +# TODO: add stage specific default configurations +}) + +class StageSpec(object): + def __init__(self, prefix, **kwargs): + self.prefix = prefix + key = ".".join([self.prefix, kwargs.get("type", "")]) + self.spec = DEFAULT_STAGE_PARAMS[key].copy() + self.spec.update(kwargs) + self.spec["type"] = key + # NOTE: special case to support reading files without passing an explicit reader + if (self.prefix == "readers") and kwargs.get("type") == "auto": + del self.spec["type"] + + @property + def pipeline(self): + output = PipelineSpec() + output.add_stage(self) + return output + + def __getattr__(self, name): + return partial(self.__class__, self.prefix, type=name) + + def __str__(self): + return json.dumps(self.spec, indent=4) + + def __add__(self, other): + return self.pipeline + other + + def execute(self): + return self.pipeline.execute() + + +readers = StageSpec("readers") +filters = StageSpec("filters") +writers = StageSpec("writers") + + +class PipelineSpec(object): + readers = [] + filters = [] + writer = None + + def __init__(self, other=None): + if other is not None: + self.readers = copy.copy(other.readers) + self.filters = copy.copy(other.filters) + self.writer = copy.copy(other.writer) + + @property + def spec(self): + return { + "pipeline": [stage.spec for stage in self.stages] + } + + @property + def stages(self): + if self.writer is not None: + return chain(self.readers, self.filters, [self.writer]) + else: + return chain(self.readers, self.filters) + + def add_stage(self, stage): + assert isinstance(stage, StageSpec), "Expected StageSpec" + if stage.prefix == "writers": + if self.writer is not None: + warnings.warn("Currently assigned output stage will be replaced.") + self.writer = stage + elif stage.prefix == "readers": + self.readers.append(stage) + elif stage.prefix == "filters": + self.filters.append(stage) + else: + warnings.warn("Unknown stage type. Skipping.") + + return self + + def __str__(self): + return json.dumps(self.spec, indent=4) + + def __add__(self, stage_or_pipeline): + assert isinstance(stage_or_pipeline, (StageSpec, PipelineSpec)), "Expected StageSpec or PipelineSpec" + + output = self.__class__(self) + if isinstance(stage_or_pipeline, StageSpec): + output.add_stage(stage_or_pipeline) + elif isinstance(stage_or_pipeline, PipelineSpec): + for stage in stage_or_pipeline.stages: + output.add_stage(stage) + return output + + def execute(self): + # TODO: do some validation before calling execute + + # TODO: some exception/error handling around pdal + pipeline = pdal.Pipeline(json.dumps(self.spec)) + # pipeline.validate() # NOTE: disabling this because it causes segfaults in certain cases + pipeline.execute() + + return pipeline.arrays[0] # NOTE: are there situation where arrays has multiple elements? From 7b9d056ff7f1f537b3431471a82610edbeae8de7 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 13:00:05 -0600 Subject: [PATCH 033/343] one basic testcase --- test/test_pio.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 test/test_pio.py diff --git a/test/test_pio.py b/test/test_pio.py new file mode 100644 index 00000000..a06b945e --- /dev/null +++ b/test/test_pio.py @@ -0,0 +1,47 @@ +import unittest +import json + +from pdal import pio + +dummy_pipeline = """{ + "pipeline": [ + { + "type": "readers.ply", + "filename": "dummyinput.ply" + }, + { + "type": "filters.outlier", + "method": "statistical", + "mean_k": 16, + "multiplier": 1.0 + }, + { + "type": "filters.range", + "limits": "Classification![7:7]" + }, + { + "type": "filters.normal" + }, + { + "type": "writers.ply", + "storage_mode": "ascii", + "precision": 4, + "filename": "dummyoutput.ply", + "dims": "X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ" + } + ] +}""" + + + +class TestPIOBasics(unittest.TestCase): + def test_pipeline_construction(self): + pipeline = (pio.readers.ply(filename="dummyfile.ply") + + pio.filters.outlier(method="statistical", mean_k=16, multiplier=1.0) + + pio.filters.range(limits="Classification![7:7]") + + pio.filters.normal() + pio.writers.ply(storage_mode="ascii", precision=4, filename=output, + dims="X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ")) + + self.assertTrue(isinstance(pipeline, pio.PipelineSpec)) + self.assertEqual(len(list(pipeline.stages)), 5) + self.assertEqual(json.dumps(pipeline.spec, indent=2, dummy_pipeline) From cc49f84c934090cf799eddd98bdcb612a384b085 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 13:54:19 -0600 Subject: [PATCH 034/343] writer doesn't need to be copied since it isn't a list --- pdal/pio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index fd1334f7..b7d6ef3b 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -56,7 +56,7 @@ def __init__(self, other=None): if other is not None: self.readers = copy.copy(other.readers) self.filters = copy.copy(other.filters) - self.writer = copy.copy(other.writer) + self.writer = other.writer @property def spec(self): From ae11bb128dfd473a2b7f8094d06c0a20a3bb0105 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 13:54:46 -0600 Subject: [PATCH 035/343] update and expand testcase --- test/test_pio.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/test_pio.py b/test/test_pio.py index a06b945e..8051e94e 100644 --- a/test/test_pio.py +++ b/test/test_pio.py @@ -36,12 +36,21 @@ class TestPIOBasics(unittest.TestCase): def test_pipeline_construction(self): - pipeline = (pio.readers.ply(filename="dummyfile.ply") + + pipeline = (pio.readers.ply(filename="dummyinput.ply") + pio.filters.outlier(method="statistical", mean_k=16, multiplier=1.0) + pio.filters.range(limits="Classification![7:7]") + - pio.filters.normal() + pio.writers.ply(storage_mode="ascii", precision=4, filename=output, + pio.filters.normal() + pio.writers.ply(storage_mode="ascii", precision=4, filename="dummyoutput.ply", dims="X,Y,Z,Red,Green,Blue,NormalX,NormalY,NormalZ")) - self.assertTrue(isinstance(pipeline, pio.PipelineSpec)) + self.assertIsInstance(pipeline, pio.PipelineSpec) self.assertEqual(len(list(pipeline.stages)), 5) - self.assertEqual(json.dumps(pipeline.spec, indent=2, dummy_pipeline) + self.assertEqual(json.dumps(pipeline.spec, indent=2), dummy_pipeline) + + pipeline = pipeline + pio.readers.auto(filename="anotherdummmyfile.laz") + + self.assertEqual(len(pipeline.readers), 2) # we can have multiple readers + + null_writer = pio.writers.null() + self.assertIsNot(pipeline.writer, null_writer) + pipeline = pipeline + null_writer + self.assertIs(pipeline.writer, null_writer) From f161adc2a7622b62dc42c21ff707e8de3567cb09 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 16:43:34 -0600 Subject: [PATCH 036/343] validation of available pdal drivers using introspected data --- pdal/pio.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index b7d6ef3b..3db6c1eb 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -1,4 +1,6 @@ +import types import json +import subprocess from functools import partial from collections import defaultdict from itertools import chain @@ -7,18 +9,24 @@ import pdal +PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout +PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) +modules = set([e["name"].split(".")[0] for e in PDAL_DRIVERS]) + + DEFAULT_STAGE_PARAMS = defaultdict(dict) DEFAULT_STAGE_PARAMS.update({ # TODO: add stage specific default configurations }) + class StageSpec(object): def __init__(self, prefix, **kwargs): self.prefix = prefix - key = ".".join([self.prefix, kwargs.get("type", "")]) - self.spec = DEFAULT_STAGE_PARAMS[key].copy() + self.key = ".".join([self.prefix, kwargs.get("type", "")]) + self.spec = DEFAULT_STAGE_PARAMS[self.key].copy() self.spec.update(kwargs) - self.spec["type"] = key + self.spec["type"] = self.key # NOTE: special case to support reading files without passing an explicit reader if (self.prefix == "readers") and kwargs.get("type") == "auto": del self.spec["type"] @@ -30,6 +38,7 @@ def pipeline(self): return output def __getattr__(self, name): + assert name in dir(self), "Invalid or unsupported stage" return partial(self.__class__, self.prefix, type=name) def __str__(self): @@ -38,6 +47,10 @@ def __str__(self): def __add__(self, other): return self.pipeline + other + def __dir__(self): + extra_keys = [e["name"][len(self.key):] for e in PDAL_DRIVERS if e["name"].startswith(self.key)] + return super().__dir__() + [e for e in extra_keys if len(e) > 0] + def execute(self): return self.pipeline.execute() From 5ee8f728edffae3da5ef0b768b0a5a01057116ed Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 16:47:17 -0600 Subject: [PATCH 037/343] raise in AttributeError instead of an AssertionError when accessing an invalid stage --- pdal/pio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index 3db6c1eb..06aadaa8 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -38,7 +38,8 @@ def pipeline(self): return output def __getattr__(self, name): - assert name in dir(self), "Invalid or unsupported stage" + if name not in dir(self): + raise AttributeError(f"'{self.prefix}.{name}' is an invalid or unsupported PDAL stage") return partial(self.__class__, self.prefix, type=name) def __str__(self): From 10e30accd8b8e972ded7a88274714bc7755a18e1 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 16:50:11 -0600 Subject: [PATCH 038/343] remove unused var --- pdal/pio.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index 06aadaa8..7d3b07f1 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -11,8 +11,6 @@ PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) -modules = set([e["name"].split(".")[0] for e in PDAL_DRIVERS]) - DEFAULT_STAGE_PARAMS = defaultdict(dict) DEFAULT_STAGE_PARAMS.update({ From 811067ec933b8890ba4e81723f6dd085d3ecaaeb Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 18 Nov 2019 17:00:33 -0600 Subject: [PATCH 039/343] make validation conditional in case pdal isn't on the path --- pdal/pio.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index 7d3b07f1..8150fa07 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -9,8 +9,13 @@ import pdal -PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout -PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) +try: + PDAL_DRIVERS_JSON = subprocess.run(["pdal", "--drivers", "--showjson"], capture_output=True).stdout + PDAL_DRIVERS = json.loads(PDAL_DRIVERS_JSON) + _PDAL_VALIDATE = True +except: + PDAL_DRIVERS = [] + _PDAL_VALIDATE = False DEFAULT_STAGE_PARAMS = defaultdict(dict) DEFAULT_STAGE_PARAMS.update({ @@ -36,7 +41,7 @@ def pipeline(self): return output def __getattr__(self, name): - if name not in dir(self): + if _PDAL_VALIDATE and (name not in dir(self)): raise AttributeError(f"'{self.prefix}.{name}' is an invalid or unsupported PDAL stage") return partial(self.__class__, self.prefix, type=name) From fd8dd0b235c2cbb76b5050350faa382f7e511e80 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 19 Nov 2019 11:27:01 -0600 Subject: [PATCH 040/343] need to include auto in extra_keys --- pdal/pio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index 8150fa07..26a4001d 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -52,7 +52,7 @@ def __add__(self, other): return self.pipeline + other def __dir__(self): - extra_keys = [e["name"][len(self.key):] for e in PDAL_DRIVERS if e["name"].startswith(self.key)] + extra_keys = [e["name"][len(self.key):] for e in PDAL_DRIVERS if e["name"].startswith(self.key)] + ["auto"] return super().__dir__() + [e for e in extra_keys if len(e) > 0] def execute(self): From 8a2f74f0a7662dc32687dec2900965c45adab1d4 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 25 Nov 2019 15:50:41 -0600 Subject: [PATCH 041/343] removing tracking of stage types --- pdal/pio.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/pdal/pio.py b/pdal/pio.py index 26a4001d..bf8cf60c 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -65,15 +65,11 @@ def execute(self): class PipelineSpec(object): - readers = [] - filters = [] - writer = None + stages = [] def __init__(self, other=None): if other is not None: - self.readers = copy.copy(other.readers) - self.filters = copy.copy(other.filters) - self.writer = other.writer + self.stages = copy.copy(other.stages) @property def spec(self): @@ -81,26 +77,10 @@ def spec(self): "pipeline": [stage.spec for stage in self.stages] } - @property - def stages(self): - if self.writer is not None: - return chain(self.readers, self.filters, [self.writer]) - else: - return chain(self.readers, self.filters) - def add_stage(self, stage): assert isinstance(stage, StageSpec), "Expected StageSpec" - if stage.prefix == "writers": - if self.writer is not None: - warnings.warn("Currently assigned output stage will be replaced.") - self.writer = stage - elif stage.prefix == "readers": - self.readers.append(stage) - elif stage.prefix == "filters": - self.filters.append(stage) - else: - warnings.warn("Unknown stage type. Skipping.") + self.stages.append(stage) return self def __str__(self): From bab8f1df61302bdf5c38362e92300452c8b6d074 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 25 Nov 2019 16:00:11 -0600 Subject: [PATCH 042/343] update test to not check for the readers/writer/filters behavior --- test/test_pio.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/test_pio.py b/test/test_pio.py index 8051e94e..de0b2471 100644 --- a/test/test_pio.py +++ b/test/test_pio.py @@ -45,12 +45,3 @@ def test_pipeline_construction(self): self.assertIsInstance(pipeline, pio.PipelineSpec) self.assertEqual(len(list(pipeline.stages)), 5) self.assertEqual(json.dumps(pipeline.spec, indent=2), dummy_pipeline) - - pipeline = pipeline + pio.readers.auto(filename="anotherdummmyfile.laz") - - self.assertEqual(len(pipeline.readers), 2) # we can have multiple readers - - null_writer = pio.writers.null() - self.assertIsNot(pipeline.writer, null_writer) - pipeline = pipeline + null_writer - self.assertIs(pipeline.writer, null_writer) From 5cf38859e73be500426978d9b8f4cff06a2e6803 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 3 Dec 2019 10:11:11 -0500 Subject: [PATCH 043/343] adding writers.auto case --- pdal/pio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pdal/pio.py b/pdal/pio.py index bf8cf60c..013003fe 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -31,7 +31,7 @@ def __init__(self, prefix, **kwargs): self.spec.update(kwargs) self.spec["type"] = self.key # NOTE: special case to support reading files without passing an explicit reader - if (self.prefix == "readers") and kwargs.get("type") == "auto": + if (self.prefix in ["readers", "writers"]) and kwargs.get("type") == "auto": del self.spec["type"] @property From 6c82060adeeb1d7b83c473c5022710551de455b4 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Tue, 3 Dec 2019 10:11:19 -0500 Subject: [PATCH 044/343] adding tests for auto readers and writers --- test/test_pio.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_pio.py b/test/test_pio.py index de0b2471..bcde6bcb 100644 --- a/test/test_pio.py +++ b/test/test_pio.py @@ -45,3 +45,13 @@ def test_pipeline_construction(self): self.assertIsInstance(pipeline, pio.PipelineSpec) self.assertEqual(len(list(pipeline.stages)), 5) self.assertEqual(json.dumps(pipeline.spec, indent=2), dummy_pipeline) + + auto_reader = pio.readers.auto(filename="dummyinput.las") + auto_writer = pio.writers.auto(filename="dummyoutput.las") + + self.assertIn("filename", auto_reader.spec) + self.assertNotIn("type", auto_reader.spec) + self.assertIn("filename", auto_reader.spec) + self.assertNotIn("type", auto_writer.spec) + self.assertEqual(auto_reader.prefix, "readers") + self.assertEqual(auto_writer.prefix, "writers") From b613647d5f12099455a74ac851b720849b534823 Mon Sep 17 00:00:00 2001 From: Pramukta Kumar Date: Mon, 9 Dec 2019 17:38:15 -0600 Subject: [PATCH 045/343] adding module docstrings --- pdal/pio.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pdal/pio.py b/pdal/pio.py index 013003fe..fbbc15e0 100644 --- a/pdal/pio.py +++ b/pdal/pio.py @@ -1,3 +1,27 @@ +""" +This module provides a python-syntax interface for constructing and executing pdal-python json +pipelines. The API is not explicitly defined but stage names are validated against the pdal executable's drivers when possible. + +To construct pipeline stages, access the driver name from this module. This will create +a callable function where driver parameters can be specified as keyword arguments. For example: + +>>> from pdal import pio +>>> las_reader = pio.readers.las(filename="test.las") + +To construct a pipeline, sum stages together. + +>>> pipeline = pio.readers.las(filename="test.las") + pio.writers.ply(filename="test.ply") + +To execute a pipeline and return results, call `execute`. + +>>> arr = pipeline.execute() # returns a numpy structured array + +To access the pipelines as a dict (which may be dumped to json), call `spec`. + +>>> json.dumps(pipeline.spec) + +""" + import types import json import subprocess @@ -36,6 +60,10 @@ def __init__(self, prefix, **kwargs): @property def pipeline(self): + """ + Promote this stage to a `pdal.pio.PipelineSpec` with one `pdal.pio.StageSpec` + and return it. + """ output = PipelineSpec() output.add_stage(self) return output @@ -73,11 +101,17 @@ def __init__(self, other=None): @property def spec(self): + """ + Return a `dict` containing the pdal pipeline suitable for dumping to json + """ return { "pipeline": [stage.spec for stage in self.stages] } def add_stage(self, stage): + """ + Add a StageSpec to the end of this pipeline, and return the updated result. + """ assert isinstance(stage, StageSpec), "Expected StageSpec" self.stages.append(stage) @@ -98,6 +132,9 @@ def __add__(self, stage_or_pipeline): return output def execute(self): + """ + Shortcut to execute and return the results of the pipeline. + """ # TODO: do some validation before calling execute # TODO: some exception/error handling around pdal From 3d94698d789f6dcb498a2cb127c06e49d1cd89e1 Mon Sep 17 00:00:00 2001 From: Andrew Bell Date: Wed, 18 Dec 2019 13:26:15 -0500 Subject: [PATCH 046/343] Make dimension number case more flexible. --- test/test_pipeline.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_pipeline.py b/test/test_pipeline.py index 823ae1ff..e16124d9 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -196,10 +196,8 @@ class TestDimensions(PDALTest): def test_fetch_dimensions(self): """Ask PDAL for its valid dimensions list""" dims = pdal.dimensions - if Version(pdal.info.version) < Version('1.8'): - self.assertEqual(len(dims), 71) - else: - self.assertEqual(len(dims), 72) + self.assertLess(len(dims), 100) + self.assertGreater(len(dims), 71) def test_suite(): return unittest.TestSuite( From 80eea96fada65f734ed4dd26a08681d1c2d5c415 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 19 Dec 2019 10:22:33 -0600 Subject: [PATCH 047/343] initial builds of readers.numpy and filters.python outside of PDAL main tree using distutils --- pdal/__init__.py | 2 +- pdal/filters/PythonFilter.cpp | 159 + pdal/filters/PythonFilter.hpp | 72 + pdal/io/NumpyReader.cpp | 520 + pdal/io/NumpyReader.hpp | 124 + pdal/nlohmann/json.hpp | 20894 ++++++++++++++++++++++++++++++++ pdal/plang/Environment.cpp | 381 + pdal/plang/Environment.hpp | 85 + pdal/plang/Invocation.cpp | 485 + pdal/plang/Invocation.hpp | 88 + pdal/plang/Redirector.cpp | 208 + pdal/plang/Redirector.hpp | 53 + pdal/plang/Script.cpp | 66 + pdal/plang/Script.hpp | 78 + setup.py | 152 +- 15 files changed, 23314 insertions(+), 53 deletions(-) create mode 100644 pdal/filters/PythonFilter.cpp create mode 100644 pdal/filters/PythonFilter.hpp create mode 100644 pdal/io/NumpyReader.cpp create mode 100644 pdal/io/NumpyReader.hpp create mode 100644 pdal/nlohmann/json.hpp create mode 100644 pdal/plang/Environment.cpp create mode 100644 pdal/plang/Environment.hpp create mode 100644 pdal/plang/Invocation.cpp create mode 100644 pdal/plang/Invocation.hpp create mode 100644 pdal/plang/Redirector.cpp create mode 100644 pdal/plang/Redirector.hpp create mode 100644 pdal/plang/Script.cpp create mode 100644 pdal/plang/Script.hpp diff --git a/pdal/__init__.py b/pdal/__init__.py index 4d775cb6..c1b548d7 100644 --- a/pdal/__init__.py +++ b/pdal/__init__.py @@ -1,4 +1,4 @@ -__version__='2.2.2' +__version__='2.3.0' from .pipeline import Pipeline from .array import Array diff --git a/pdal/filters/PythonFilter.cpp b/pdal/filters/PythonFilter.cpp new file mode 100644 index 00000000..6b8f5399 --- /dev/null +++ b/pdal/filters/PythonFilter.cpp @@ -0,0 +1,159 @@ +/****************************************************************************** +* Copyright (c) 2011, Michael P. Gerlek (mpg@flaxen.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "PythonFilter.hpp" + +#include "../nlohmann/json.hpp" + +#include +#include +#include +#include + +namespace pdal +{ + +static PluginInfo const s_info +{ + "filters.python", + "Manipulate data using inline Python", + "http://pdal.io/stages/filters.python.html" +}; + +CREATE_SHARED_STAGE(PythonFilter, s_info) + +struct PythonFilter::Args +{ + std::string m_module; + std::string m_function; + std::string m_source; + std::string m_scriptFile; + StringList m_addDimensions; + NL::json m_pdalargs; +}; + +PythonFilter::PythonFilter() : + m_script(nullptr), m_pythonMethod(nullptr), m_args(new Args) +{} + + +PythonFilter::~PythonFilter() +{} + + +std::string PythonFilter::getName() const +{ + return s_info.name; +} + + +void PythonFilter::addArgs(ProgramArgs& args) +{ + args.add("module", "Python module containing the function to run", + m_args->m_module).setPositional(); + args.add("function", "Function to call", + m_args->m_function).setPositional(); + args.add("source", "Python script to run", m_args->m_source); + args.add("script", "File containing script to run", m_args->m_scriptFile); + args.add("add_dimension", "Dimensions to add", m_args->m_addDimensions); + args.add("pdalargs", "Dictionary to add to module globals when " + "calling function", m_args->m_pdalargs); +} + + +void PythonFilter::addDimensions(PointLayoutPtr layout) +{ + for (const std::string& s : m_args->m_addDimensions) + { + StringList spec = Utils::split(s, '='); + Utils::trim(spec[0]); + if (spec.size() == 2) + { + Utils::trim(spec[1]); + auto type = pdal::Dimension::type(spec[1]); + if (type == pdal::Dimension::Type::None) + throwError("Invalid dimension type specified '" + spec[1] + + "'. See documentation for valid dimension types"); + layout->registerOrAssignDim(spec[0], type); + } + else if (spec.size() == 1) + layout->registerOrAssignDim(s, pdal::Dimension::Type::Double); + else + throwError("Invalid dimension specified '" + s + + "'. Need or =."); + } +} + + +void PythonFilter::prepared(PointTableRef table) +{ + if (m_args->m_source.size() && m_args->m_scriptFile.size()) + throwError("Can't set both 'source' and 'script' options."); + if (!m_args->m_source.size() && !m_args->m_scriptFile.size()) + throwError("Must set one of 'source' and 'script' options."); +} + + +void PythonFilter::ready(PointTableRef table) +{ + if (m_args->m_source.empty()) + m_args->m_source = FileUtils::readFileIntoString(m_args->m_scriptFile); + std::ostream *out = log()->getLogStream(); + plang::EnvironmentPtr env = plang::Environment::get(); + env->set_stdout(out); + m_script.reset(new plang::Script(m_args->m_source, m_args->m_module, + m_args->m_function)); + m_pythonMethod.reset(new plang::Invocation(*m_script, table.metadata(), + m_args->m_pdalargs.dump(1))); +} + + +PointViewSet PythonFilter::run(PointViewPtr view) +{ + log()->get(LogLevel::Debug5) << "filters.python " << *m_script << + " processing " << view->size() << " points." << std::endl; + m_pythonMethod->execute(view, getMetadata()); + + PointViewSet viewSet; + viewSet.insert(view); + return viewSet; +} + + +void PythonFilter::done(PointTableRef table) +{ + static_cast(plang::Environment::get())->reset_stdout(); +} + +} // namespace pdal diff --git a/pdal/filters/PythonFilter.hpp b/pdal/filters/PythonFilter.hpp new file mode 100644 index 00000000..19f3d036 --- /dev/null +++ b/pdal/filters/PythonFilter.hpp @@ -0,0 +1,72 @@ +/****************************************************************************** +* Copyright (c) 2011, Michael P. Gerlek (mpg@flaxen.com) +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include +#include + +#include "../plang/Invocation.hpp" + + +namespace pdal +{ + +class PDAL_DLL PythonFilter : public Filter +{ +public: + PythonFilter(); + virtual ~PythonFilter(); + + std::string getName() const; + +private: + PythonFilter& operator=(const PythonFilter&) = delete; + PythonFilter(const PythonFilter&) = delete; + + virtual void addArgs(ProgramArgs& args); + virtual void addDimensions(PointLayoutPtr layout); + virtual void prepared(PointTableRef table); + virtual void ready(PointTableRef table); + virtual PointViewSet run(PointViewPtr view); + virtual void done(PointTableRef table); + + std::unique_ptr m_script; + std::unique_ptr m_pythonMethod; + + struct Args; + std::unique_ptr m_args; +}; + +} // namespace pdal diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp new file mode 100644 index 00000000..d67e1fc9 --- /dev/null +++ b/pdal/io/NumpyReader.cpp @@ -0,0 +1,520 @@ +/****************************************************************************** +* Copyright (c) 2018, Howard Butler, howard@hobu.co +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#include "NumpyReader.hpp" + +#include +#include +#include +#include +#include +#include + +#include "../plang/Environment.hpp" + + +std::string toString(PyObject *pname) +{ + std::stringstream mssg; + PyObject* r = PyObject_Str(pname); + if (!r) + throw pdal::pdal_error("couldn't make string representation value"); +#if PY_MAJOR_VERSION >= 3 + Py_ssize_t size; + const char *d = PyUnicode_AsUTF8AndSize(r, &size); +#else + const char *d = PyString_AsString(r); +#endif + mssg << d; + return mssg.str(); +} + + +namespace pdal +{ + +static PluginInfo const s_info +{ + "readers.numpy", + "Read data from .npy files.", + "" +}; + +CREATE_SHARED_STAGE(NumpyReader, s_info) + +std::string NumpyReader::getName() const { return s_info.name; } + +std::ostream& operator << (std::ostream& out, + const NumpyReader::Order& order) +{ + switch (order) + { + case NumpyReader::Order::Row: + out << "row"; + break; + case NumpyReader::Order::Column: + out << "column"; + break; + } + return out; +} + + +std::istream& operator >> (std::istream& in, NumpyReader::Order& order) +{ + std::string s; + in >> s; + + s = Utils::tolower(s); + if (s == "row") + order = NumpyReader::Order::Row; + else if (s == "column") + order = NumpyReader::Order::Column; + else + in.setstate(std::ios_base::failbit); + return in; +} + + +void NumpyReader::setArray(PyObject* array) +{ + plang::Environment::get(); + if (!PyArray_Check(array)) + throw pdal::pdal_error("object provided to setArray is not a python numpy array!"); + + m_array = (PyArrayObject*)array; + Py_XINCREF(m_array); +} + +PyArrayObject* load_npy(std::string const& filename) +{ + + PyObject *py_filename = PyUnicode_FromString(filename.c_str()); + PyObject *numpy_module = PyImport_ImportModule("numpy"); + if (!numpy_module) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject *numpy_mod_dict = PyModule_GetDict(numpy_module); + if (!numpy_mod_dict) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject *loads_func = PyDict_GetItemString(numpy_mod_dict, "load"); + if (!loads_func) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject *numpy_args = PyTuple_New(1); + if (!numpy_args) + throw pdal::pdal_error(plang::getTraceback()); + + if (PyTuple_SetItem(numpy_args, 0, py_filename)) + throw pdal::pdal_error(plang::getTraceback()); + + PyObject* array = PyObject_CallObject(loads_func, numpy_args); + if (!array) + throw pdal::pdal_error(plang::getTraceback()); + + return reinterpret_cast(array); +} + + +void NumpyReader::initialize() +{ + plang::Environment::get(); + m_numPoints = 0; + m_chunkCount = 0; + m_ndims = 0; + + m_iter = NULL; + + p_data = NULL; + m_dataptr = NULL; + m_strideptr = NULL; + m_innersizeptr = NULL; + m_dtype = NULL; + + if (m_filename.size()) + { + m_array = load_npy(m_filename); + } + if (m_array) + if (!PyArray_Check(m_array)) + throw pdal::pdal_error("Object in file '" + m_filename + + "' is not a numpy array"); +} + + +void NumpyReader::wakeUpNumpyArray() +{ + // TODO pivot whether we are a 1d, 2d, or named arrays + + if (PyArray_SIZE(m_array) == 0) + throw pdal::pdal_error("Array cannot be empty!"); + + m_iter = NpyIter_New(m_array, + NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY| NPY_ITER_REFS_OK, + NPY_KEEPORDER, NPY_NO_CASTING, + NULL); + if (!m_iter) + { + std::ostringstream oss; + + oss << "Unable to create iterator from array in '" + << m_filename + "' with traceback: '" + << plang::getTraceback() <<"'"; + throw pdal::pdal_error(oss.str()); + } + + char* itererr; + m_iternext = NpyIter_GetIterNext(m_iter, &itererr); + if (!m_iternext) + { + NpyIter_Deallocate(m_iter); + throw pdal::pdal_error(itererr); + } + + m_dtype = PyArray_DTYPE(m_array); + if (!m_dtype) + throw pdal_error(plang::getTraceback()); + + m_ndims = PyArray_NDIM(m_array); + m_shape = PyArray_SHAPE(m_array); + if (!m_shape) + throw pdal_error(plang::getTraceback()); + + npy_intp* shape = PyArray_SHAPE(m_array); + if (!shape) + throw pdal_error(plang::getTraceback()); + m_numPoints = 1; + for (int i = 0; i < m_ndims; ++i) + m_numPoints *= m_shape[i]; + + // If the order arg wasn't set, set order based on the internal order of + // the array. + if (!m_orderArg->set()) + { + int flags = PyArray_FLAGS(m_array); + if (flags & NPY_ARRAY_F_CONTIGUOUS) + m_order = Order::Column; + else + m_order = Order::Row; + } +} + + +void NumpyReader::addArgs(ProgramArgs& args) +{ + args.add("dimension", "In an unstructured array, the dimension name to " + "map to values.", m_defaultDimension, "Intensity"); + m_orderArg = &args.add("order", "Order of dimension interpretation " + "of the array. Either 'row'-major (C) or 'column'-major (Fortran)", + m_order); +} + + +// Try to map dimension names to existing PDAL dimension names by +// checking them with certain characters removed. +Dimension::Id NumpyReader::registerDim(PointLayoutPtr layout, + const std::string& name, Dimension::Type pdalType) +{ + Dimension::Id id; + + auto registerName = [&layout, &pdalType, &id](std::string name, char elim) + { + if (elim != '\0') + Utils::remove(name, elim); + Dimension::Id tempId = Dimension::id(name); + if (tempId != Dimension::Id::Unknown) + { + id = tempId; + layout->registerDim(id, pdalType); + return true; + } + return false; + }; + + // Try registering the name in various ways. If that doesn't work, + // just punt and use the name as is. + if (!registerName(name, '\0') && !registerName(name, '-') && + !registerName(name, ' ') && !registerName(name, '_')) + id = layout->registerOrAssignDim(name, pdalType); + return id; +} + +namespace +{ + + +Dimension::Type getType(PyArray_Descr *dtype, const std::string& name) +{ + if (!dtype) + throw pdal_error("Can't fetch data type for numpy field."); + + Dimension::Type pdalType = + plang::Environment::getPDALDataType(dtype->type_num); + if (pdalType == Dimension::Type::None) + { + std::ostringstream oss; + oss << "Unable to map dimension '" << name << "' because its " + "type '" << dtype->type_num <<"' is not mappable to PDAL"; + throw pdal_error(oss.str()); + } + return pdalType; +} + +} // unnamed namespace + + +void NumpyReader::createFields(PointLayoutPtr layout) +{ + Dimension::Id id; + Dimension::Type type; + int offset; + + m_numFields = 0; + if (m_dtype->fields != Py_None) + m_numFields = static_cast(PyDict_Size(m_dtype->fields)); + + // Array isn't structured - just a bunch of data. + if (m_numFields <= 0) + { + type = getType(m_dtype, m_defaultDimension); + id = registerDim(layout, m_defaultDimension, type); + m_fields.push_back({id, type, 0}); + } + else + { + PyObject* names_dict = m_dtype->fields; + PyObject* names = PyDict_Keys(names_dict); + PyObject* values = PyDict_Values(names_dict); + if (!names || !values) + throw pdal_error("Bad field specification for numpy array layout."); + + for (int i = 0; i < m_numFields; ++i) + { + std::string name = toString(PyList_GetItem(names, i)); + PyObject *tup = PyList_GetItem(values, i); + if (!tup) + throw pdal_error(plang::getTraceback()); + + // Get offset. + PyObject* offset_o = PySequence_Fast_GET_ITEM(tup, 1); + if (!offset_o) + throw pdal_error(plang::getTraceback()); + offset = PyLong_AsLong(offset_o); + + // Get type. + type = getType((PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0), + name); + id = registerDim(layout, name, type); + m_fields.push_back({id, type, offset}); + } + } +} + + +void NumpyReader::addDimensions(PointLayoutPtr layout) +{ + using namespace Dimension; + + wakeUpNumpyArray(); + createFields(layout); + + m_storeXYZ = true; + // If we already have an X dimension, we're done. + for (const Field& field : m_fields) + if (field.m_id == Id::X || field.m_id == Id::Y || field.m_id == Id::Z) + { + m_storeXYZ = false; + return; + } + + // We're storing a calculated XYZ, so register the dims. + layout->registerDim(Id::X, Type::Signed32); + if (m_ndims > 1) + { + layout->registerDim(Id::Y, Type::Signed32); + if (m_ndims > 2) + layout->registerDim(Id::Z, Type::Signed32); + } + if (m_order == Order::Row) + { + m_xIter = m_shape[m_ndims - 1]; + m_xDiv = 1; + if (m_ndims > 1) + { + m_yDiv = 1; + m_xDiv = m_xIter; + m_xIter *= m_shape[m_ndims - 2]; + m_yIter = m_shape[m_ndims - 1]; + if (m_ndims > 2) + { + m_xDiv = m_xIter; + m_yDiv = m_yIter; + m_zDiv = 1; + m_xIter *= m_shape[m_ndims - 3]; + m_yIter *= m_shape[m_ndims - 2]; + m_zIter = m_shape[m_ndims - 1]; + } + } + } + else + { + m_xIter = m_shape[0]; + m_xDiv = 1; + if (m_ndims > 1) + { + m_yIter = m_shape[0] * m_shape[1]; + m_yDiv = m_xIter; + if (m_ndims > 2) + { + m_zIter = m_shape[0] * m_shape[1] * m_shape[2]; + m_zDiv = m_yIter; + } + } + } +} + + +void NumpyReader::ready(PointTableRef table) +{ + plang::Environment::get()->set_stdout(log()->getLogStream()); + + // Set our iterators + // The location of the data pointer which the iterator may update + m_dataptr = NpyIter_GetDataPtrArray(m_iter); + + // The location of the stride which the iterator may update + m_strideptr = NpyIter_GetInnerStrideArray(m_iter); + + // The location of the inner loop size which the iterator may update + m_innersizeptr = NpyIter_GetInnerLoopSizePtr(m_iter); + + p_data = *m_dataptr; + m_chunkCount = *m_innersizeptr; + m_index = 0; + + log()->get(LogLevel::Debug) << "Initializing Numpy array for file '" << + m_filename << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy inner stride '" << + *m_strideptr << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy inner stride size '" << + *m_innersizeptr << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy number of points '" << + m_numPoints << "'" << std::endl; + log()->get(LogLevel::Debug) << "numpy number of dimensions '" << + m_ndims << "'" << std::endl; + for (npy_intp i = 0; i < m_ndims; ++i) + log()->get(LogLevel::Debug) << "numpy shape dimension number '" << + i << "' is '" << m_shape[i] <<"'" << std::endl; +} + +bool NumpyReader::nextPoint() +{ + // If we are at the end of this chunk, try to fetch another. Otherwise, + // just advance by the stride. + if (--m_chunkCount == 0) + { + // If we can't fetch the next ite + if (!m_iternext(m_iter)) + return false; + m_chunkCount = *m_innersizeptr; + p_data = *m_dataptr; + } + else + p_data += *m_strideptr; + return true; +} + + +bool NumpyReader::loadPoint(PointRef& point, point_count_t position) +{ + using namespace Dimension; + + for (const Field& f : m_fields) + point.setField(f.m_id, f.m_type, (void*)(p_data + f.m_offset)); + + if (m_storeXYZ) + { + point.setField(Dimension::Id::X, (position % m_xIter) / m_xDiv); + if (m_ndims > 1) + { + point.setField(Dimension::Id::Y, (position % m_yIter) / m_yDiv); + if (m_ndims > 2) + point.setField(Dimension::Id::Z, (position % m_zIter) / m_zDiv); + } + } + + return (nextPoint()); +} + + +bool NumpyReader::processOne(PointRef& point) +{ + if (m_index >= m_numPoints) + return false; + return loadPoint(point, m_index++); +} + + +point_count_t NumpyReader::read(PointViewPtr view, point_count_t numToRead) +{ + PointId idx = view->size(); + point_count_t numRead(0); + + while (numRead < numToRead) + { + PointRef point(*view, idx); + if (!processOne(point)) + break; + numRead++; + idx++; + } + return numRead; +} + + +void NumpyReader::done(PointTableRef) +{ + // Dereference everything we're using + + if (m_iter) + NpyIter_Deallocate(m_iter); + + Py_XDECREF(m_array); +} + + +} // namespace pdal + + diff --git a/pdal/io/NumpyReader.hpp b/pdal/io/NumpyReader.hpp new file mode 100644 index 00000000..31a18102 --- /dev/null +++ b/pdal/io/NumpyReader.hpp @@ -0,0 +1,124 @@ +/****************************************************************************** +* Copyright (c) 2018, Howard Butler, howard@hobu.co +* +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following +* conditions are met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided +* with the distribution. +* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the +* names of its contributors may be used to endorse or promote +* products derived from this software without specific prior +* written permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +* OF SUCH DAMAGE. +****************************************************************************/ + +#pragma once + +#include +#include + +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#define PY_ARRAY_UNIQUE_SYMBOL PDAL_ARRAY_API +#define NO_IMPORT_ARRAY +#include + +namespace pdal +{ + +class PDAL_DLL NumpyReader : public Reader, public Streamable +{ + enum class Order + { + Row, + Column + }; + friend std::ostream& operator << (std::ostream& out, + const NumpyReader::Order& order); + friend std::istream& operator >> (std::istream& in, + NumpyReader::Order& order); +public: + NumpyReader& operator=(const NumpyReader&) = delete; + NumpyReader(const NumpyReader&) = delete; + NumpyReader() : m_array(NULL) + {} + + void setArray(PyObject* array); + std::string getName() const; + +private: + virtual void initialize(); + virtual void addArgs(ProgramArgs& args); + virtual void addDimensions(PointLayoutPtr layout); + virtual void ready(PointTableRef table); + virtual point_count_t read(PointViewPtr view, point_count_t count); + virtual bool processOne(PointRef& point); + virtual void done(PointTableRef table); + + void createFields(PointLayoutPtr layout); + bool nextPoint(); + bool loadPoint(PointRef& point, point_count_t position); + void wakeUpNumpyArray(); + Dimension::Id registerDim(PointLayoutPtr layout, const std::string& name, + Dimension::Type pdalType); + void prepareFieldsArray(PointLayoutPtr layout); + void prepareRasterArray(PointLayoutPtr layout); + + // Py_XDECREF these on the way out + PyArrayObject* m_array; + NpyIter* m_iter; + NpyIter_IterNextFunc* m_iternext; + PyArray_Descr* m_dtype; + + char** m_dataptr; + char* p_data; + npy_intp m_nonzero_count; + npy_intp* m_strideptr, *m_innersizeptr; + npy_intp* m_shape; + npy_intp m_chunkCount; + point_count_t m_numPoints; + int m_numFields; + + Arg *m_orderArg; + int m_ndims; + std::string m_defaultDimension; + Order m_order; + bool m_storeXYZ; + size_t m_xIter; + size_t m_yIter; + size_t m_zIter; + size_t m_xDiv; + size_t m_yDiv; + size_t m_zDiv; + + struct Field + { + Dimension::Id m_id; + Dimension::Type m_type; + int m_offset; + }; + std::vector m_fields; + point_count_t m_index; +}; + +} // namespace pdal diff --git a/pdal/nlohmann/json.hpp b/pdal/nlohmann/json.hpp new file mode 100644 index 00000000..3c28a9d6 --- /dev/null +++ b/pdal/nlohmann/json.hpp @@ -0,0 +1,20894 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.6.1 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2019 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 6 +#define NLOHMANN_JSON_VERSION_PATCH 1 + +#include // all_of, find, for_each +#include // assert +#include // and, not, or +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#include // istream, ostream +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include + + +#include + +// #include + + +#include // transform +#include // array +#include // and, not +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include + + +#include // exception +#include // runtime_error +#include // to_string + +// #include + + +#include // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// + +/*! +@brief general exception of the @ref basic_json class + +This class is an extension of `std::exception` objects with a member @a id for +exception ids. It is used as the base class for all exceptions thrown by the +@ref basic_json class. This class can hence be used as "wildcard" to catch +exceptions. + +Subclasses: +- @ref parse_error for exceptions indicating a parse error +- @ref invalid_iterator for exceptions indicating errors with iterators +- @ref type_error for exceptions indicating executing a member function with + a wrong type +- @ref out_of_range for exceptions indicating access out of the defined range +- @ref other_error for exceptions indicating other library errors + +@internal +@note To have nothrow-copy-constructible exceptions, we internally use + `std::runtime_error` which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. +@endinternal + +@liveexample{The following code shows how arbitrary library exceptions can be +caught.,exception} + +@since version 3.0.0 +*/ +class exception : public std::exception +{ + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + +/*! +@brief exception indicating a parse error + +This exception is thrown by the library when a parse error occurs. Parse errors +can occur during the deserialization of JSON text, CBOR, MessagePack, as well +as when using JSON Patch. + +Member @a byte holds the byte index of the last read character in the input +file. + +Exceptions have ids 1xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). + +@note For an input with n bytes, 1 is the index of the first character and n+1 + is the index of the terminating null byte or the end of file. This also + holds true when reading a byte vector (CBOR or MessagePack). + +@liveexample{The following code shows how a `parse_error` exception can be +caught.,parse_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] pos the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } + + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id_, byte_, w.c_str()); + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; + +/*! +@brief exception indicating errors with iterators + +This exception is thrown if iterators passed to a library function do not match +the expected semantics. + +Exceptions have ids 2xx. + +name / id | example message | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@liveexample{The following code shows how an `invalid_iterator` exception can be +caught.,invalid_iterator} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id_, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id_) + what_arg; + return invalid_iterator(id_, w.c_str()); + } + + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating executing a member function with a wrong type + +This exception is thrown in case of a type error; that is, a library function is +executed on a JSON value whose type does not match the expected semantics. + +Exceptions have ids 3xx. + +name / id | example message | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t &. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. +json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | + +@liveexample{The following code shows how a `type_error` exception can be +caught.,type_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class type_error : public exception +{ + public: + static type_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("type_error", id_) + what_arg; + return type_error(id_, w.c_str()); + } + + private: + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating access out of the defined range + +This exception is thrown in case a library function is called on an input +parameter that exceeds the expected range, for instance in case of array +indices or nonexisting object keys. + +Exceptions have ids 4xx. + +name / id | example message | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | + +@liveexample{The following code shows how an `out_of_range` exception can be +caught.,out_of_range} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class out_of_range : public exception +{ + public: + static out_of_range create(int id_, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id_) + what_arg; + return out_of_range(id_, w.c_str()); + } + + private: + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating other library errors + +This exception is thrown in case of errors that cannot be classified with the +other exception types. + +Exceptions have ids 5xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range + +@liveexample{The following code shows how an `other_error` exception can be +caught.,other_error} + +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("other_error", id_) + what_arg; + return other_error(id_, w.c_str()); + } + + private: + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // pair + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +// allow for portable nodiscard warnings +#if defined(__has_cpp_attribute) + #if __has_cpp_attribute(nodiscard) + #if defined(__clang__) && !defined(JSON_HAS_CPP_17) // issue #1535 + #define JSON_NODISCARD + #else + #define JSON_NODISCARD [[nodiscard]] + #endif + #elif __has_cpp_attribute(gnu::warn_unused_result) + #define JSON_NODISCARD [[gnu::warn_unused_result]] + #else + #define JSON_NODISCARD + #endif +#else + #define JSON_NODISCARD +#endif + +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// manual branch prediction +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_LIKELY(x) __builtin_expect(x, 1) + #define JSON_UNLIKELY(x) __builtin_expect(x, 0) +#else + #define JSON_LIKELY(x) x + #define JSON_UNLIKELY(x) x +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// #include + + +#include // not +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +template +using uncvref_t = typename std::remove_cv::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template +struct merge_and_renumber; + +template +struct merge_and_renumber, index_sequence> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template +using index_sequence_for = make_index_sequence; + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // not +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval + +// #include + + +#include // random_access_iterator_tag + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + + +#include + +// #include + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template