Skip to content

feat: N-dimensional raster type extension (Phase 1) #749

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
james-willis wants to merge 15 commits into
base: main
Choose a base branch
from
Open

feat: N-dimensional raster type extension (Phase 1) #749

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
c205066
refactor(sedona-schema): N-D raster schema
james-willis Apr 1, 2026
f44920e
refactor(sedona-raster): N-D traits, array reader, builder
james-willis Apr 1, 2026
2c707cd
refactor(sedona-raster, sedona-testing): N-D traits, reader, builder,…
james-willis Apr 1, 2026
c027101
refactor(sedona-raster-functions): migrate RS_* functions to N-D API
james-willis Apr 1, 2026
5576420
test(sedona-raster): add N-D capability tests
james-willis Apr 1, 2026
d6a18e5
refactor: add BandDataType::try_from_u32, deduplicate conversion
james-willis Apr 1, 2026
567c887
test: add unit tests for BandDataType::try_from_u32
james-willis Apr 1, 2026
01f2f43
fix: resolve clippy warnings and formatting
james-willis Apr 1, 2026
8aebf60
fix: rename variable to satisfy codespell pre-commit check
james-willis Apr 2, 2026
57a3760
feat: add outdb_uri support to builder, restore outdb tests
james-willis Apr 2, 2026
218b5f3
fix: use scheme-dispatched URIs in outdb test data
james-willis Apr 2, 2026
ae00bbd
docs: fix BandRef trait doc to reflect stride-aware access via nd_buffer
james-willis Apr 2, 2026
40ad8c4
docs: clarify contiguous_data() is zero-copy for standard strides
james-willis Apr 2, 2026
66b3c2e
fix: address code review issues
james-willis Apr 2, 2026
f4c36c7
feat: add outdb_uri parser, RS_BandPath returns plain path
james-willis Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 24 additions & 15 deletions rust/sedona-raster-functions/src/executor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,13 +359,16 @@ impl<'a, 'b> RasterExecutor<'a, 'b> {
arr0.len()
);
}

// Hoist the RasterStructArray so its lifetime covers the loop.
let scalar_arr1;
let r1 = match sv1 {
ScalarValue::Struct(arc_struct) => {
let arr1 = RasterStructArray::new(arc_struct.as_ref());
if arr1.is_null(0) {
scalar_arr1 = RasterStructArray::new(arc_struct.as_ref());
if scalar_arr1.is_null(0) {
None
} else {
Some(arr1.get(0)?)
Some(scalar_arr1.get(0)?)
}
}
ScalarValue::Null => None,
Expand Down Expand Up @@ -396,13 +399,16 @@ impl<'a, 'b> RasterExecutor<'a, 'b> {
arr1.len()
);
}

// Hoist the RasterStructArray so its lifetime covers the loop.
let scalar_arr0;
let r0 = match sv0 {
ScalarValue::Struct(arc_struct) => {
let arr0 = RasterStructArray::new(arc_struct.as_ref());
if arr0.is_null(0) {
scalar_arr0 = RasterStructArray::new(arc_struct.as_ref());
if scalar_arr0.is_null(0) {
None
} else {
Some(arr0.get(0)?)
Some(scalar_arr0.get(0)?)
}
}
ScalarValue::Null => None,
Expand All @@ -422,27 +428,30 @@ impl<'a, 'b> RasterExecutor<'a, 'b> {
Ok(())
}
(ColumnarValue::Scalar(sv0), ColumnarValue::Scalar(sv1)) => {
// Hoist both RasterStructArrays so their lifetimes cover the loop.
let scalar_arr0;
let r0 = match sv0 {
ScalarValue::Struct(arc_struct) => {
let arr0 = RasterStructArray::new(arc_struct.as_ref());
if arr0.is_null(0) {
scalar_arr0 = RasterStructArray::new(arc_struct.as_ref());
if scalar_arr0.is_null(0) {
None
} else {
Some(arr0.get(0)?)
Some(scalar_arr0.get(0)?)
}
}
ScalarValue::Null => None,
_ => {
return sedona_internal_err!("Expected Struct scalar for raster");
}
};
let scalar_arr1;
let r1 = match sv1 {
ScalarValue::Struct(arc_struct) => {
let arr1 = RasterStructArray::new(arc_struct.as_ref());
if arr1.is_null(0) {
scalar_arr1 = RasterStructArray::new(arc_struct.as_ref());
if scalar_arr1.is_null(0) {
None
} else {
Some(arr1.get(0)?)
Some(scalar_arr1.get(0)?)
}
}
ScalarValue::Null => None,
Expand Down Expand Up @@ -725,7 +734,7 @@ mod tests {
match raster_opt {
None => builder.append_null(),
Some(raster) => {
let width = raster.metadata().width();
let width = raster.width().unwrap();
builder.append_value(width);
}
}
Expand Down Expand Up @@ -767,7 +776,7 @@ mod tests {
match raster_opt {
None => builder.append_null(),
Some(raster) => {
let width = raster.metadata().width();
let width = raster.width().unwrap();
builder.append_value(width);
}
}
Expand Down Expand Up @@ -804,7 +813,7 @@ mod tests {
match raster_opt {
None => builder.append_null(),
Some(raster) => {
let width = raster.metadata().width();
let width = raster.width().unwrap();
builder.append_value(width);
}
}
Expand Down
69 changes: 31 additions & 38 deletions rust/sedona-raster-functions/src/rs_band_accessors.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,13 +120,18 @@ fn get_pixel_type(
Ok(())
}
Some(raster) => {
let num_bands = raster.bands().len();
let num_bands = raster.num_bands();
if band_index < 1 || band_index > num_bands as i32 {
builder.append_null();
return Ok(());
}
let band = raster.bands().band(band_index as usize)?;
let dt = band.metadata().data_type()?;
let band = raster.band((band_index - 1) as usize).ok_or_else(|| {
datafusion_common::DataFusionError::Internal(format!(
"Band index {} out of range",
band_index
))
})?;
let dt = band.data_type();
builder.append_value(dt.pixel_type_name());
Ok(())
}
Expand Down Expand Up @@ -224,14 +229,21 @@ fn get_nodata_value(
Ok(())
}
Some(raster) => {
let num_bands = raster.bands().len();
let num_bands = raster.num_bands();
if band_index < 1 || band_index > num_bands as i32 {
builder.append_null();
return Ok(());
}
let band = raster.bands().band(band_index as usize)?;
let band_meta = band.metadata();
match band_meta.nodata_value_as_f64()? {
let band = raster.band((band_index - 1) as usize).ok_or_else(|| {
datafusion_common::DataFusionError::Internal(format!(
"Band index {} out of range",
band_index
))
})?;
match band
.nodata_as_f64()
.map_err(datafusion_common::DataFusionError::from)?
{
None => builder.append_null(),
Some(val) => builder.append_value(val),
}
Expand All @@ -246,30 +258,27 @@ mod tests {
use arrow_array::{Array, Float64Array, Int32Array, Int64Array, StringArray, StructArray};
use datafusion_expr::ScalarUDF;
use sedona_raster::builder::RasterBuilder;
use sedona_raster::traits::{BandMetadata, RasterMetadata};
use sedona_schema::datatypes::RASTER;
use sedona_schema::raster::{BandDataType, StorageType};
use sedona_schema::raster::BandDataType;
use sedona_testing::compare::assert_array_equal;
use sedona_testing::rasters::generate_test_rasters;
use sedona_testing::testers::ScalarUdfTester;

/// Build a single-row raster StructArray with custom metadata and band metadata.
/// Build a single-row raster StructArray with custom parameters.
fn build_custom_raster(
meta: &RasterMetadata,
band_meta: &BandMetadata,
width: u64,
height: u64,
data_type: BandDataType,
nodata: Option<&[u8]>,
data: &[u8],
crs: Option<&str>,
) -> StructArray {
let mut builder = RasterBuilder::new(1);
builder.start_raster(meta, crs).expect("start raster");
builder
.start_band(BandMetadata {
datatype: band_meta.datatype,
nodata_value: band_meta.nodata_value.clone(),
storage_type: band_meta.storage_type,
outdb_url: band_meta.outdb_url.clone(),
outdb_band_id: band_meta.outdb_band_id,
})
.start_raster_2d(width, height, 0.0, 0.0, 1.0, -1.0, 0.0, 0.0, crs)
.expect("start raster");
builder
.start_band_2d(data_type, nodata)
.expect("start band");
builder.band_data_writer().append_value(data);
builder.finish_band().expect("finish band");
Expand Down Expand Up @@ -401,25 +410,9 @@ mod tests {
#[test]
fn udf_bandnodatavalue_no_nodata() {
// Create a raster without nodata
let meta = RasterMetadata {
width: 2,
height: 2,
upperleft_x: 0.0,
upperleft_y: 0.0,
scale_x: 1.0,
scale_y: -1.0,
skew_x: 0.0,
skew_y: 0.0,
};
let band_meta = BandMetadata {
datatype: BandDataType::UInt8,
nodata_value: None,
storage_type: StorageType::InDb,
outdb_url: None,
outdb_band_id: None,
};
let data = vec![1u8, 2, 3, 4];
let rasters = build_custom_raster(&meta, &band_meta, &data, Some("OGC:CRS84"));
let rasters =
build_custom_raster(2, 2, BandDataType::UInt8, None, &data, Some("OGC:CRS84"));

let udf: ScalarUDF = rs_bandnodatavalue_udf().into();
let tester = ScalarUdfTester::new(udf, vec![RASTER]);
Expand Down
101 changes: 43 additions & 58 deletions rust/sedona-raster-functions/src/rs_bandpath.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ use datafusion_common::error::Result;
use datafusion_expr::{ColumnarValue, Volatility};
use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF};
use sedona_raster::traits::RasterRef;
use sedona_schema::raster::StorageType;
use sedona_schema::{datatypes::SedonaType, matchers::ArgMatcher};

/// RS_BandPath() scalar UDF implementation
Expand Down Expand Up @@ -124,22 +123,23 @@ fn get_band_path(
match raster_opt {
None => builder.append_null(),
Some(raster) => {
let bands = raster.bands();
let num_bands = bands.len() as i32;
let num_bands = raster.num_bands() as i32;
if band_index < 1 || band_index > num_bands {
builder.append_null();
} else {
let band = bands.band(band_index as usize)?;
let band_metadata = band.metadata();

if band_metadata.storage_type()? == StorageType::OutDbRef {
match band_metadata.outdb_url() {
Some(url) => builder.append_value(url),
None => builder.append_null(),
} else if let Some(band) = raster.band((band_index - 1) as usize) {
match band.outdb_uri() {
Some(uri) => {
// Return just the path portion, stripping the internal
// scheme prefix and fragment from the outdb_uri.
let path = sedona_raster::outdb_uri::parse_outdb_uri(uri)
.map(|parsed| parsed.path)
.unwrap_or(uri);
builder.append_value(path);
}
} else {
builder.append_null()
None => builder.append_null(),
}
} else {
builder.append_null();
}
}
}
Expand Down Expand Up @@ -225,11 +225,11 @@ mod tests {
.downcast_ref::<StringArray>()
.expect("Expected StringArray");

// Raster 0, band 1: OutDbRef -> URL
// Raster 0, band 1: OutDbRef → URI
assert_eq!(string_array.value(0), "s3://bucket/raster_0.tif");
// Raster 1: null raster -> null
// Raster 1: null raster → null
assert!(string_array.is_null(1));
// Raster 2, band 2: OutDbRef -> URL
// Raster 2, band 2: OutDbRef → URI
assert_eq!(string_array.value(2), "s3://bucket/raster_2.tif");
}

Expand Down Expand Up @@ -258,37 +258,28 @@ mod tests {

/// Build a raster array with out-db bands for testing RS_BandPath.
/// Returns a StructArray with 3 rasters:
/// [0] OutDbRef band with URL "s3://bucket/raster_0.tif"
/// [0] OutDbRef band with URI "geotiff://s3://bucket/raster_0.tif#band=1"
/// [1] null raster
/// [2] Two bands: InDb band 1, OutDbRef band 2 with URL "s3://bucket/raster_2.tif"
/// [2] Two bands: InDb band 1, OutDbRef band 2 with URI "geotiff://s3://bucket/raster_2.tif#band=3"
fn build_outdb_rasters() -> arrow_array::StructArray {
use sedona_raster::builder::RasterBuilder;
use sedona_raster::traits::{BandMetadata, RasterMetadata};
use sedona_schema::raster::{BandDataType, StorageType};

let metadata = RasterMetadata {
width: 4,
height: 4,
upperleft_x: 0.0,
upperleft_y: 0.0,
scale_x: 1.0,
scale_y: -1.0,
skew_x: 0.0,
skew_y: 0.0,
};
use sedona_schema::raster::BandDataType;

let mut builder = RasterBuilder::new(3);

// Raster 0: single OutDbRef band
builder.start_raster(&metadata, Some("EPSG:4326")).unwrap();
builder
.start_band(BandMetadata {
nodata_value: None,
storage_type: StorageType::OutDbRef,
datatype: BandDataType::Float32,
outdb_url: Some("s3://bucket/raster_0.tif".to_string()),
outdb_band_id: Some(1),
})
.start_raster_2d(4, 4, 0.0, 0.0, 1.0, -1.0, 0.0, 0.0, Some("EPSG:4326"))
.unwrap();
builder
.start_band(
None,
&["y", "x"],
&[4, 4],
BandDataType::Float32,
None,
Some("geotiff://s3://bucket/raster_0.tif#band=1"),
)
.unwrap();
builder.band_data_writer().append_value([]);
builder.finish_band().unwrap();
Expand All @@ -298,26 +289,21 @@ mod tests {
builder.append_null().unwrap();

// Raster 2: two bands — InDb (band 1) + OutDbRef (band 2)
builder.start_raster(&metadata, Some("EPSG:4326")).unwrap();
builder
.start_band(BandMetadata {
nodata_value: None,
storage_type: StorageType::InDb,
datatype: BandDataType::UInt8,
outdb_url: None,
outdb_band_id: None,
})
.start_raster_2d(4, 4, 0.0, 0.0, 1.0, -1.0, 0.0, 0.0, Some("EPSG:4326"))
.unwrap();
builder.start_band_2d(BandDataType::UInt8, None).unwrap();
builder.band_data_writer().append_value([0u8; 16]);
builder.finish_band().unwrap();
builder
.start_band(BandMetadata {
nodata_value: None,
storage_type: StorageType::OutDbRef,
datatype: BandDataType::Float32,
outdb_url: Some("s3://bucket/raster_2.tif".to_string()),
outdb_band_id: Some(3),
})
.start_band(
None,
&["y", "x"],
&[4, 4],
BandDataType::Float32,
None,
Some("geotiff://s3://bucket/raster_2.tif#band=3"),
)
.unwrap();
builder.band_data_writer().append_value([]);
builder.finish_band().unwrap();
Expand All @@ -339,8 +325,7 @@ mod tests {
.downcast_ref::<StringArray>()
.expect("Expected StringArray");

// Raster 0: OutDbRef band 1 → returns URL
assert!(!string_array.is_null(0));
// Raster 0: OutDbRef band → URI
assert_eq!(string_array.value(0), "s3://bucket/raster_0.tif");
// Raster 1: null raster → null
assert!(string_array.is_null(1));
Expand All @@ -365,11 +350,11 @@ mod tests {
.downcast_ref::<StringArray>()
.expect("Expected StringArray");

// Raster 0, band 1: OutDbRef → URL
// Raster 0, band 1: OutDbRef → URI
assert_eq!(string_array.value(0), "s3://bucket/raster_0.tif");
// Raster 1: null raster → null
assert!(string_array.is_null(1));
// Raster 2, band 2: OutDbRef → URL
// Raster 2, band 2: OutDbRef → URI
assert_eq!(string_array.value(2), "s3://bucket/raster_2.tif");
}

Expand Down
Loading
Loading