Fixed #32670 -- Allowed GDALRasters to use any GDAL virtual filesystem.

django/contrib/gis/gdal/raster/const.py

@@ -65,8 +65,11 @@ GDAL_COLOR_TYPES = {
     16: 'GCI_YCbCr_CrBand',  # Cr Chroma, also GCI_Max
 }
 
+# GDAL virtual filesystems prefix.
+VSI_FILESYSTEM_PREFIX = '/vsi'
+
 # Fixed base path for buffer-based GDAL in-memory files.
-VSI_FILESYSTEM_BASE_PATH = '/vsimem/'
+VSI_MEM_FILESYSTEM_BASE_PATH = '/vsimem/'
 
 # Should the memory file system take ownership of the buffer, freeing it when
 # the file is deleted? (No, GDALRaster.__del__() will delete the buffer.)

django/contrib/gis/gdal/raster/source.py

@@ -12,8 +12,8 @@ from django.contrib.gis.gdal.prototypes import raster as capi
 from django.contrib.gis.gdal.raster.band import BandList
 from django.contrib.gis.gdal.raster.base import GDALRasterBase
 from django.contrib.gis.gdal.raster.const import (
-    GDAL_RESAMPLE_ALGORITHMS, VSI_DELETE_BUFFER_ON_READ,
+    GDAL_RESAMPLE_ALGORITHMS, VSI_DELETE_BUFFER_ON_READ, VSI_FILESYSTEM_PREFIX,
-    VSI_FILESYSTEM_BASE_PATH, VSI_TAKE_BUFFER_OWNERSHIP,
+    VSI_MEM_FILESYSTEM_BASE_PATH, VSI_TAKE_BUFFER_OWNERSHIP,
 )
 from django.contrib.gis.gdal.srs import SpatialReference, SRSException
 from django.contrib.gis.geometry import json_regex
@@ -74,7 +74,7 @@ class GDALRaster(GDALRasterBase):
         # If input is a valid file path, try setting file as source.
         if isinstance(ds_input, str):
             if (
-                not ds_input.startswith(VSI_FILESYSTEM_BASE_PATH) and
+                not ds_input.startswith(VSI_FILESYSTEM_PREFIX) and
                 not os.path.exists(ds_input)
             ):
                 raise GDALException(
@@ -95,7 +95,7 @@ class GDALRaster(GDALRasterBase):
             # deleted.
             self._ds_input = c_buffer(ds_input)
             # Create random name to reference in vsimem filesystem.
-            vsi_path = os.path.join(VSI_FILESYSTEM_BASE_PATH, str(uuid.uuid4()))
+            vsi_path = os.path.join(VSI_MEM_FILESYSTEM_BASE_PATH, str(uuid.uuid4()))
             # Create vsimem file from buffer.
             capi.create_vsi_file_from_mem_buffer(
                 force_bytes(vsi_path),
@@ -217,7 +217,10 @@ class GDALRaster(GDALRasterBase):
 
     @property
     def vsi_buffer(self):
-        if not self.is_vsi_based:
+        if not (
+            self.is_vsi_based and
+            self.name.startswith(VSI_MEM_FILESYSTEM_BASE_PATH)
+        ):
             return None
         # Prepare an integer that will contain the buffer length.
         out_length = c_int()
@@ -232,7 +235,7 @@ class GDALRaster(GDALRasterBase):
 
     @cached_property
     def is_vsi_based(self):
-        return self._ptr and self.name.startswith(VSI_FILESYSTEM_BASE_PATH)
+        return self._ptr and self.name.startswith(VSI_FILESYSTEM_PREFIX)
 
     @property
     def name(self):
@@ -432,7 +435,7 @@ class GDALRaster(GDALRasterBase):
         elif self.driver.name != 'MEM':
             clone_name = self.name + '_copy.' + self.driver.name
         else:
-            clone_name = os.path.join(VSI_FILESYSTEM_BASE_PATH, str(uuid.uuid4()))
+            clone_name = os.path.join(VSI_MEM_FILESYSTEM_BASE_PATH, str(uuid.uuid4()))
         return GDALRaster(
             capi.copy_ds(
                 self.driver._ptr,

docs/ref/contrib/gis/gdal.txt

@@ -1111,9 +1111,9 @@ blue.
     raster should be opened in write mode. For newly-created rasters, the second
     parameter is ignored and the new raster is always created in write mode.
 
-    The first parameter can take three forms: a string representing a file
+    The first parameter can take three forms: a string representing a file path
-    path, a dictionary with values defining a new raster, or a bytes object
+    (filesystem or GDAL virtual filesystem), a dictionary with values defining
-    representing a raster file.
+    a new raster, or a bytes object representing a raster file.
 
     If the input is a file path, the raster is opened from there. If the input
     is raw data in a dictionary, the parameters ``width``, ``height``, and
@@ -1164,6 +1164,10 @@ blue.
         >>> rst.name  # Stored in a random path in the vsimem filesystem.
         '/vsimem/da300bdb-129d-49a8-b336-e410a9428dad'
 
+    .. versionchanged:: 4.0
+
+        Creating rasters in any GDAL virtual filesystem was allowed.
+
     .. attribute:: name
 
         The name of the source which is equivalent to the input file path or the name
@@ -1772,6 +1776,13 @@ Key              Default                           Usage
 Using GDAL's Virtual Filesystem
 -------------------------------
 
+GDAL can access files stored in the filesystem, but also supports virtual
+filesystems to abstract accessing other kind of files, such as compressed,
+encrypted, or remote files.
+
+Using memory-based Virtual Filesystem
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 GDAL has an internal memory-based filesystem, which allows treating blocks of
 memory as files. It can be used to read and write :class:`GDALRaster` objects
 to and from binary file buffers.
@@ -1817,6 +1828,53 @@ Here's how to create a raster and return it as a file in an
     ... })
     >>> HttpResponse(rast.vsi_buffer, 'image/tiff')
 
+Using other Virtual Filesystems
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 4.0
+
+Depending on the local build of GDAL other virtual filesystems may be
+supported. You can use them by prepending the provided path with the
+appropriate ``/vsi*/`` prefix. See the `GDAL Virtual Filesystems
+documentation`_ for more details.
+
+.. warning:
+
+    Rasters with names starting with `/vsi*/` will be treated as rasters from
+    the GDAL virtual filesystems. Django doesn't perform any extra validation.
+
+Compressed rasters
+^^^^^^^^^^^^^^^^^^
+
+Instead decompressing the file and instantiating the resulting raster, GDAL can
+directly access compressed files using the ``/vsizip/``, ``/vsigzip/``, or
+``/vsitar/`` virtual filesystems::
+
+   >>> from django.contrib.gis.gdal import GDALRaster
+   >>> rst = GDALRaster('/vsizip/path/to/your/file.zip/path/to/raster.tif')
+   >>> rst = GDALRaster('/vsigzip/path/to/your/file.gz')
+   >>> rst = GDALRaster('/vsitar/path/to/your/file.tar/path/to/raster.tif')
+
+Network rasters
+^^^^^^^^^^^^^^^
+
+GDAL can support online resources and storage providers transparently. As long
+as it's built with such capabilities.
+
+To access a public raster file with no authentication, you can use
+``/vsicurl/``::
+
+   >>> from django.contrib.gis.gdal import GDALRaster
+   >>> rst = GDALRaster('/vsicurl/https://example.com/raster.tif')
+   >>> rst.name
+   '/vsicurl/https://example.com/raster.tif'
+
+For commercial storage providers (e.g. ``/vsis3/``) the system should be
+previously configured for authentication and possibly other settings (see the
+`GDAL Virtual Filesystems documentation`_ for available options).
+
+.. _`GDAL Virtual Filesystems documentation`: https://gdal.org/user/virtual_file_systems.html
+
 Settings
 ========
 

docs/releases/4.0.txt

@@ -102,6 +102,9 @@ Minor features
 
 * Added support for SpatiaLite 5.
 
+* :class:`~django.contrib.gis.gdal.GDALRaster` now allows creating rasters in
+  any GDAL virtual filesystem.
+
 :mod:`django.contrib.messages`
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/spelling_wordlist

@@ -217,6 +217,7 @@ fieldsets
 filename
 filenames
 filesystem
+filesystems
 fk
 flatpage
 flatpages

tests/gis_tests/gdal_tests/test_raster.py

@@ -2,6 +2,7 @@ import os
 import shutil
 import struct
 import tempfile
+import zipfile
 from unittest import mock
 
 from django.contrib.gis.gdal import GDALRaster, SpatialReference
@@ -229,6 +230,17 @@ class GDALRasterTests(SimpleTestCase):
         # The vsi buffer is None for rasters that are not vsi based.
         self.assertIsNone(self.rs.vsi_buffer)
 
+    def test_vsi_vsizip_filesystem(self):
+        rst_zipfile = tempfile.NamedTemporaryFile(suffix='.zip')
+        with zipfile.ZipFile(rst_zipfile, mode='w') as zf:
+            zf.write(self.rs_path, 'raster.tif')
+        rst_path = '/vsizip/' + os.path.join(rst_zipfile.name, 'raster.tif')
+        rst = GDALRaster(rst_path)
+        self.assertEqual(rst.driver.name, self.rs.driver.name)
+        self.assertEqual(rst.name, rst_path)
+        self.assertIs(rst.is_vsi_based, True)
+        self.assertIsNone(rst.vsi_buffer)
+
     def test_offset_size_and_shape_on_raster_creation(self):
         rast = GDALRaster({
             'datatype': 1,