diff --git a/csv-core/src/reader.rs b/csv-core/src/reader.rs index 5bcbdfa..b38d609 100644 --- a/csv-core/src/reader.rs +++ b/csv-core/src/reader.rs @@ -877,7 +877,7 @@ impl Reader { if ends.is_empty() { return (ReadRecordResult::OutputEndsFull, 0, 0, 0); } - let (mut nin, mut nout, mut nend) = (0, self.output_pos, 0); + let (mut nin, mut nout, mut nend) = (0, 0, 0); let mut state = self.nfa_state; while nin < input.len() && nout < output.len() && nend < ends.len() { let (s, io) = self.transition_nfa(state, input[nin]); @@ -894,7 +894,7 @@ impl Reader { } state = s; if state.is_field_final() { - ends[nend] = nout; + ends[nend] = self.output_pos + nout; nend += 1; if state != NfaState::EndFieldDelim { break; @@ -908,7 +908,11 @@ impl Reader { nend >= ends.len(), ); self.nfa_state = state; - self.output_pos = if res.is_record() { 0 } else { nout }; + if res.is_record() { + self.output_pos = 0; + } else { + self.output_pos += nout; + } (res, nin, nout, nend) } diff --git a/csv-core/tests/repro_nfa_split.rs b/csv-core/tests/repro_nfa_split.rs new file mode 100644 index 0000000..589e524 --- /dev/null +++ b/csv-core/tests/repro_nfa_split.rs @@ -0,0 +1,48 @@ +use csv_core::{ReadRecordResult, ReaderBuilder}; + +fn read(nfa: bool, chunks: &[&[u8]]) -> Vec { + let mut rdr = ReaderBuilder::new().nfa(nfa).build(); + let mut out = [0u8; 64]; + let mut ends = [0usize; 8]; + let (mut outpos, mut endpos) = (0, 0); + let mut fields = vec![]; + let mut chunks = chunks.iter().copied().chain(std::iter::once(&b""[..])); + let mut data = chunks.next().unwrap(); + loop { + let (res, nin, nout, nend) = + rdr.read_record(data, &mut out[outpos..], &mut ends[endpos..]); + data = &data[nin..]; + outpos += nout; + endpos += nend; + match res { + ReadRecordResult::InputEmpty => match chunks.next() { + Some(c) => data = c, + None => break, + }, + ReadRecordResult::Record => { + let mut start = 0; + for &end in &ends[..endpos] { + fields.push( + String::from_utf8_lossy(&out[start..end]).into_owned(), + ); + start = end; + } + outpos = 0; + endpos = 0; + } + ReadRecordResult::End => break, + r => panic!("{r:?}"), + } + } + fields +} + +#[test] +fn repro_nfa_record_split_across_calls() { + let chunks: &[&[u8]] = &[b"ab,c", b"d,ef\n"]; + let dfa = read(false, chunks); + let nfa = read(true, chunks); + println!("dfa={dfa:?}\nnfa={nfa:?}"); + assert_eq!(dfa, ["ab", "cd", "ef"]); + assert_eq!(nfa, dfa); +}