Skip to content

fix(writer): skip NaN min/max when aggregating Parquet column bounds #3539

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevinjqliu merged 3 commits into from
Oct 27, 2026
+41 −2
Merged

fix(writer): skip NaN min/max when aggregating Parquet column bounds #3539

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd65083
fix(writer): skip NaN min/max when aggregating Parquet column bounds
andygrove Oct 26, 2026
3ab570d
docs(writer): clarify why NaN bounds are skipped
andygrove Oct 26, 2026
b82e58c
fix(writer): correct NaN bound comment and cover negative NaN lower b…
kevinjqliu Oct 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 41 additions & 2 deletions crates/iceberg/src/writer/file_writer/parquet_writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,10 @@ impl MinMaxColAggregator {
));
};

self.update_state_min(field_id, min_datum);
// Iceberg bounds must not be NaN.
if !min_datum.is_nan() {
self.update_state_min(field_id, min_datum);
}
Comment on lines +383 to +385

@andygrove andygrove Oct 26, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Leaving this as is. Each guard is a single !is_nan() check, so a helper would save one line per call site and add indirection. If more invalid-bound rules appear, a shared predicate makes sense then.

}

if value.max_is_exact() {
Expand All @@ -390,7 +393,9 @@ impl MinMaxColAggregator {
));
};

self.update_state_max(field_id, max_datum);
if !max_datum.is_nan() {
self.update_state_max(field_id, max_datum);
}
Comment on lines +396 to +398

@andygrove andygrove Oct 26, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as the min case above: leaving the guard inline.

}

Ok(())
Expand Down Expand Up @@ -2605,6 +2610,40 @@ mod tests {
assert_eq!(upper_bounds, HashMap::from([(0, Datum::int(i32::MAX))]));
}

#[test]
fn test_min_max_aggregator_skips_nan_bounds() {
let schema = Arc::new(
Schema::builder()
.with_schema_id(1)
.with_fields(vec![
NestedField::required(0, "f", Type::Primitive(PrimitiveType::Float)).into(),
NestedField::required(1, "d", Type::Primitive(PrimitiveType::Double)).into(),
])
.build()
.expect("Failed to create schema"),
);
let mut min_max_agg = MinMaxColAggregator::new(schema);
let float_stats =
|min, max| Statistics::Float(ValueStatistics::new(min, max, None, None, false));
let double_stats =
|min, max| Statistics::Double(ValueStatistics::new(min, max, None, None, false));

min_max_agg
.update(0, &float_stats(Some(-f32::NAN), Some(f32::NAN)))
.unwrap();
min_max_agg
.update(0, &float_stats(Some(1.5), Some(2.5)))
.unwrap();
min_max_agg
.update(1, &double_stats(Some(f64::NAN), Some(f64::NAN)))
.unwrap();

let (lower_bounds, upper_bounds) = min_max_agg.produce();

assert_eq!(lower_bounds, HashMap::from([(0, Datum::float(1.5_f32))]));
assert_eq!(upper_bounds, HashMap::from([(0, Datum::float(2.5_f32))]));
}

fn cdc_test_schema() -> SchemaRef {
Arc::new(
Schema::builder()
Expand Down
Loading