From 205c36b58fed5a1a0ff462593fc61b58189027d8 Mon Sep 17 00:00:00 2001 From: Jordi Castells Date: Wed, 14 Apr 2021 16:07:36 +0200 Subject: [PATCH] Fixed #32670 -- Allowed GDALRasters to use any GDAL virtual filesystem. --- django/contrib/gis/gdal/raster/const.py | 5 +- django/contrib/gis/gdal/raster/source.py | 17 +++--- docs/ref/contrib/gis/gdal.txt | 64 +++++++++++++++++++++-- docs/releases/4.0.txt | 3 ++ docs/spelling_wordlist | 1 + tests/gis_tests/gdal_tests/test_raster.py | 12 +++++ 6 files changed, 91 insertions(+), 11 deletions(-) diff --git a/django/contrib/gis/gdal/raster/const.py b/django/contrib/gis/gdal/raster/const.py index f9793e6213b..83636cf05ea 100644 --- a/django/contrib/gis/gdal/raster/const.py +++ b/django/contrib/gis/gdal/raster/const.py @@ -65,8 +65,11 @@ GDAL_COLOR_TYPES = { 16: 'GCI_YCbCr_CrBand', # Cr Chroma, also GCI_Max } +# GDAL virtual filesystems prefix. +VSI_FILESYSTEM_PREFIX = '/vsi' + # Fixed base path for buffer-based GDAL in-memory files. -VSI_FILESYSTEM_BASE_PATH = '/vsimem/' +VSI_MEM_FILESYSTEM_BASE_PATH = '/vsimem/' # Should the memory file system take ownership of the buffer, freeing it when # the file is deleted? (No, GDALRaster.__del__() will delete the buffer.) diff --git a/django/contrib/gis/gdal/raster/source.py b/django/contrib/gis/gdal/raster/source.py index 4d2f0dd9483..7c17b859258 100644 --- a/django/contrib/gis/gdal/raster/source.py +++ b/django/contrib/gis/gdal/raster/source.py @@ -12,8 +12,8 @@ from django.contrib.gis.gdal.prototypes import raster as capi from django.contrib.gis.gdal.raster.band import BandList from django.contrib.gis.gdal.raster.base import GDALRasterBase from django.contrib.gis.gdal.raster.const import ( - GDAL_RESAMPLE_ALGORITHMS, VSI_DELETE_BUFFER_ON_READ, - VSI_FILESYSTEM_BASE_PATH, VSI_TAKE_BUFFER_OWNERSHIP, + GDAL_RESAMPLE_ALGORITHMS, VSI_DELETE_BUFFER_ON_READ, VSI_FILESYSTEM_PREFIX, + VSI_MEM_FILESYSTEM_BASE_PATH, VSI_TAKE_BUFFER_OWNERSHIP, ) from django.contrib.gis.gdal.srs import SpatialReference, SRSException from django.contrib.gis.geometry import json_regex @@ -74,7 +74,7 @@ class GDALRaster(GDALRasterBase): # If input is a valid file path, try setting file as source. if isinstance(ds_input, str): if ( - not ds_input.startswith(VSI_FILESYSTEM_BASE_PATH) and + not ds_input.startswith(VSI_FILESYSTEM_PREFIX) and not os.path.exists(ds_input) ): raise GDALException( @@ -95,7 +95,7 @@ class GDALRaster(GDALRasterBase): # deleted. self._ds_input = c_buffer(ds_input) # Create random name to reference in vsimem filesystem. - vsi_path = os.path.join(VSI_FILESYSTEM_BASE_PATH, str(uuid.uuid4())) + vsi_path = os.path.join(VSI_MEM_FILESYSTEM_BASE_PATH, str(uuid.uuid4())) # Create vsimem file from buffer. capi.create_vsi_file_from_mem_buffer( force_bytes(vsi_path), @@ -217,7 +217,10 @@ class GDALRaster(GDALRasterBase): @property def vsi_buffer(self): - if not self.is_vsi_based: + if not ( + self.is_vsi_based and + self.name.startswith(VSI_MEM_FILESYSTEM_BASE_PATH) + ): return None # Prepare an integer that will contain the buffer length. out_length = c_int() @@ -232,7 +235,7 @@ class GDALRaster(GDALRasterBase): @cached_property def is_vsi_based(self): - return self._ptr and self.name.startswith(VSI_FILESYSTEM_BASE_PATH) + return self._ptr and self.name.startswith(VSI_FILESYSTEM_PREFIX) @property def name(self): @@ -432,7 +435,7 @@ class GDALRaster(GDALRasterBase): elif self.driver.name != 'MEM': clone_name = self.name + '_copy.' + self.driver.name else: - clone_name = os.path.join(VSI_FILESYSTEM_BASE_PATH, str(uuid.uuid4())) + clone_name = os.path.join(VSI_MEM_FILESYSTEM_BASE_PATH, str(uuid.uuid4())) return GDALRaster( capi.copy_ds( self.driver._ptr, diff --git a/docs/ref/contrib/gis/gdal.txt b/docs/ref/contrib/gis/gdal.txt index e3d8b70ced8..7d5a205c1e3 100644 --- a/docs/ref/contrib/gis/gdal.txt +++ b/docs/ref/contrib/gis/gdal.txt @@ -1111,9 +1111,9 @@ blue. raster should be opened in write mode. For newly-created rasters, the second parameter is ignored and the new raster is always created in write mode. - The first parameter can take three forms: a string representing a file - path, a dictionary with values defining a new raster, or a bytes object - representing a raster file. + The first parameter can take three forms: a string representing a file path + (filesystem or GDAL virtual filesystem), a dictionary with values defining + a new raster, or a bytes object representing a raster file. If the input is a file path, the raster is opened from there. If the input is raw data in a dictionary, the parameters ``width``, ``height``, and @@ -1164,6 +1164,10 @@ blue. >>> rst.name # Stored in a random path in the vsimem filesystem. '/vsimem/da300bdb-129d-49a8-b336-e410a9428dad' + .. versionchanged:: 4.0 + + Creating rasters in any GDAL virtual filesystem was allowed. + .. attribute:: name The name of the source which is equivalent to the input file path or the name @@ -1772,6 +1776,13 @@ Key Default Usage Using GDAL's Virtual Filesystem ------------------------------- +GDAL can access files stored in the filesystem, but also supports virtual +filesystems to abstract accessing other kind of files, such as compressed, +encrypted, or remote files. + +Using memory-based Virtual Filesystem +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + GDAL has an internal memory-based filesystem, which allows treating blocks of memory as files. It can be used to read and write :class:`GDALRaster` objects to and from binary file buffers. @@ -1817,6 +1828,53 @@ Here's how to create a raster and return it as a file in an ... }) >>> HttpResponse(rast.vsi_buffer, 'image/tiff') +Using other Virtual Filesystems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 4.0 + +Depending on the local build of GDAL other virtual filesystems may be +supported. You can use them by prepending the provided path with the +appropriate ``/vsi*/`` prefix. See the `GDAL Virtual Filesystems +documentation`_ for more details. + +.. warning: + + Rasters with names starting with `/vsi*/` will be treated as rasters from + the GDAL virtual filesystems. Django doesn't perform any extra validation. + +Compressed rasters +^^^^^^^^^^^^^^^^^^ + +Instead decompressing the file and instantiating the resulting raster, GDAL can +directly access compressed files using the ``/vsizip/``, ``/vsigzip/``, or +``/vsitar/`` virtual filesystems:: + + >>> from django.contrib.gis.gdal import GDALRaster + >>> rst = GDALRaster('/vsizip/path/to/your/file.zip/path/to/raster.tif') + >>> rst = GDALRaster('/vsigzip/path/to/your/file.gz') + >>> rst = GDALRaster('/vsitar/path/to/your/file.tar/path/to/raster.tif') + +Network rasters +^^^^^^^^^^^^^^^ + +GDAL can support online resources and storage providers transparently. As long +as it's built with such capabilities. + +To access a public raster file with no authentication, you can use +``/vsicurl/``:: + + >>> from django.contrib.gis.gdal import GDALRaster + >>> rst = GDALRaster('/vsicurl/https://example.com/raster.tif') + >>> rst.name + '/vsicurl/https://example.com/raster.tif' + +For commercial storage providers (e.g. ``/vsis3/``) the system should be +previously configured for authentication and possibly other settings (see the +`GDAL Virtual Filesystems documentation`_ for available options). + +.. _`GDAL Virtual Filesystems documentation`: https://gdal.org/user/virtual_file_systems.html + Settings ======== diff --git a/docs/releases/4.0.txt b/docs/releases/4.0.txt index ee3aa6cc70c..45dc67203de 100644 --- a/docs/releases/4.0.txt +++ b/docs/releases/4.0.txt @@ -102,6 +102,9 @@ Minor features * Added support for SpatiaLite 5. +* :class:`~django.contrib.gis.gdal.GDALRaster` now allows creating rasters in + any GDAL virtual filesystem. + :mod:`django.contrib.messages` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/spelling_wordlist b/docs/spelling_wordlist index c65722cd5b4..a1ace59a0ff 100644 --- a/docs/spelling_wordlist +++ b/docs/spelling_wordlist @@ -217,6 +217,7 @@ fieldsets filename filenames filesystem +filesystems fk flatpage flatpages diff --git a/tests/gis_tests/gdal_tests/test_raster.py b/tests/gis_tests/gdal_tests/test_raster.py index 6858abc4d89..229f09eaf4d 100644 --- a/tests/gis_tests/gdal_tests/test_raster.py +++ b/tests/gis_tests/gdal_tests/test_raster.py @@ -2,6 +2,7 @@ import os import shutil import struct import tempfile +import zipfile from unittest import mock from django.contrib.gis.gdal import GDALRaster, SpatialReference @@ -229,6 +230,17 @@ class GDALRasterTests(SimpleTestCase): # The vsi buffer is None for rasters that are not vsi based. self.assertIsNone(self.rs.vsi_buffer) + def test_vsi_vsizip_filesystem(self): + rst_zipfile = tempfile.NamedTemporaryFile(suffix='.zip') + with zipfile.ZipFile(rst_zipfile, mode='w') as zf: + zf.write(self.rs_path, 'raster.tif') + rst_path = '/vsizip/' + os.path.join(rst_zipfile.name, 'raster.tif') + rst = GDALRaster(rst_path) + self.assertEqual(rst.driver.name, self.rs.driver.name) + self.assertEqual(rst.name, rst_path) + self.assertIs(rst.is_vsi_based, True) + self.assertIsNone(rst.vsi_buffer) + def test_offset_size_and_shape_on_raster_creation(self): rast = GDALRaster({ 'datatype': 1,