Skip to content

Commit e8874c9

Browse files
tomasz-tylenda-sonarsourcetomasz-tylenda-sonarsource
authored
SONARJAVA-5655 Extract BatchGenerator into a top-level class. (#5218)
1 parent f8a41be commit e8874c9

File tree

5 files changed

+251
-172
lines changed

5 files changed

+251
-172
lines changed

‎java-frontend/src/main/java/org/sonar/java/BatchGenerator.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* SonarQube Java
3+
* Copyright (C) 2012-2025 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12+
* See the Sonar Source-Available License for more details.
13+
*
14+
* You should have received a copy of the Sonar Source-Available License
15+
* along with this program; if not, see https://sonarsource.com/license/ssal/
16+
*/
17+
package org.sonar.java;
18+
19+
import java.util.ArrayList;
20+
import java.util.Iterator;
21+
import java.util.List;
22+
import org.sonar.api.batch.fs.InputFile;
23+
24+
class BatchGenerator {
25+
public final long batchSizeInBytes;
26+
private final Iterator<InputFile> source;
27+
private InputFile buffer = null;
28+
29+
public BatchGenerator(Iterator<InputFile> source, long batchSizeInBytes) {
30+
this.source = source;
31+
this.batchSizeInBytes = batchSizeInBytes;
32+
}
33+
34+
public boolean hasNext() {
35+
return buffer != null || source.hasNext();
36+
}
37+
38+
public List<InputFile> next() {
39+
List<InputFile> batch = clearBuffer();
40+
long batchSize = batch.isEmpty() ? 0L : batch.get(0).file().length();
41+
while (source.hasNext() && batchSize <= batchSizeInBytes) {
42+
buffer = source.next();
43+
batchSize += buffer.file().length();
44+
if (batchSize > batchSizeInBytes) {
45+
// If the batch is empty, we clear the value from the buffer and add it to the batch
46+
if (batch.isEmpty()) {
47+
batch.add(buffer);
48+
buffer = null;
49+
}
50+
// If the last inputFile does not fit into the non-empty batch, we keep it in the buffer for the next call
51+
return batch;
52+
}
53+
batch.add(buffer);
54+
}
55+
buffer = null;
56+
return batch;
57+
}
58+
59+
private List<InputFile> clearBuffer() {
60+
if (buffer == null) {
61+
return new ArrayList<>();
62+
}
63+
List<InputFile> batch = new ArrayList<>();
64+
batch.add(buffer);
65+
buffer = null;
66+
return batch;
67+
}
68+
}

‎java-frontend/src/main/java/org/sonar/java/JavaFrontend.java‎

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import java.util.Collection;
2323
import java.util.Collections;
2424
import java.util.HashSet;
25-
import java.util.Iterator;
2625
import java.util.List;
2726
import java.util.Map;
2827
import java.util.Set;
@@ -324,53 +323,6 @@ public void endOfAnalysis() {
324323

325324
}
326325

327-
static class BatchGenerator {
328-
public final long batchSizeInBytes;
329-
private final Iterator<InputFile> source;
330-
private InputFile buffer = null;
331-
332-
333-
public BatchGenerator(Iterator<InputFile> source, long batchSizeInBytes) {
334-
this.source = source;
335-
this.batchSizeInBytes = batchSizeInBytes;
336-
}
337-
338-
public boolean hasNext() {
339-
return buffer != null || source.hasNext();
340-
}
341-
342-
public List<InputFile> next() {
343-
List<InputFile> batch = clearBuffer();
344-
long batchSize = batch.isEmpty() ? 0L : batch.get(0).file().length();
345-
while (source.hasNext() && batchSize <= batchSizeInBytes) {
346-
buffer = source.next();
347-
batchSize += buffer.file().length();
348-
if (batchSize > batchSizeInBytes) {
349-
// If the batch is empty, we clear the value from the buffer and add it to the batch
350-
if (batch.isEmpty()) {
351-
batch.add(buffer);
352-
buffer = null;
353-
}
354-
// If the last inputFile does not fit into the non-empty batch, we keep it in the buffer for the next call
355-
return batch;
356-
}
357-
batch.add(buffer);
358-
}
359-
buffer = null;
360-
return batch;
361-
}
362-
363-
private List<InputFile> clearBuffer() {
364-
if (buffer == null) {
365-
return new ArrayList<>();
366-
}
367-
List<InputFile> batch = new ArrayList<>();
368-
batch.add(buffer);
369-
buffer = null;
370-
return batch;
371-
}
372-
}
373-
374326
@VisibleForTesting
375327
boolean isFileByFileEnabled() {
376328
return sonarComponents != null && sonarComponents.isFileByFileEnabled();

‎java-frontend/src/test/java/org/sonar/java/BatchGeneratorTest.java‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* SonarQube Java
3+
* Copyright (C) 2012-2025 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12+
* See the Sonar Source-Available License for more details.
13+
*
14+
* You should have received a copy of the Sonar Source-Available License
15+
* along with this program; if not, see https://sonarsource.com/license/ssal/
16+
*/
17+
package org.sonar.java;
18+
19+
import java.io.File;
20+
import java.io.IOException;
21+
import java.util.ArrayList;
22+
import java.util.Arrays;
23+
import java.util.Collections;
24+
import java.util.List;
25+
import org.junit.Rule;
26+
import org.junit.jupiter.api.Test;
27+
import org.junit.jupiter.migrationsupport.rules.EnableRuleMigrationSupport;
28+
import org.junit.rules.TemporaryFolder;
29+
import org.sonar.api.batch.fs.InputFile;
30+
import org.sonar.api.batch.sensor.internal.SensorContextTester;
31+
import org.sonar.api.config.internal.MapSettings;
32+
33+
import static org.assertj.core.api.Assertions.assertThat;
34+
import static org.sonar.java.InputFileUtils.addFile;
35+
36+
@EnableRuleMigrationSupport
37+
class BatchGeneratorTest {
38+
@Rule
39+
public TemporaryFolder temp = new TemporaryFolder();
40+
41+
@Test
42+
void batch_generator_returns_an_empty_list_when_no_input_files() {
43+
List<InputFile> emptyList = Collections.emptyList();
44+
BatchGenerator generator = new BatchGenerator(emptyList.iterator(), 0);
45+
assertThat(generator.hasNext()).isFalse();
46+
assertThat(generator.next()).isEmpty();
47+
}
48+
49+
@Test
50+
void batch_generator_returns_at_most_one_item_per_batch_when_size_is_zero() throws IOException {
51+
File baseDir = temp.getRoot().getAbsoluteFile();
52+
SensorContextTester sensorContext = SensorContextTester.create(baseDir);
53+
sensorContext.setSettings(new MapSettings());
54+
List<InputFile> inputFiles = new ArrayList<>();
55+
inputFiles.add(addFile(temp, "class A {}", sensorContext));
56+
inputFiles.add(addFile(temp, "class B extends A {}", sensorContext));
57+
BatchGenerator generator = new BatchGenerator(inputFiles.iterator(), 0);
58+
assertThat(generator.hasNext()).isTrue();
59+
assertThat(generator.next())
60+
.hasSize(1)
61+
.contains(inputFiles.get(0));
62+
assertThat(generator.hasNext()).isTrue();
63+
assertThat(generator.next())
64+
.hasSize(1)
65+
.contains(inputFiles.get(1));
66+
assertThat(generator.hasNext()).isFalse();
67+
assertThat(generator.next()).isEmpty();
68+
}
69+
70+
@Test
71+
void batch_generator_returns_batches_with_multiple_files_that_are_smaller_than_batch_size() throws IOException {
72+
File baseDir = temp.getRoot().getAbsoluteFile();
73+
SensorContextTester sensorContext = SensorContextTester.create(baseDir);
74+
sensorContext.setSettings(new MapSettings());
75+
InputFile fileA = addFile(temp, "class A { public void doSomething() {} }", sensorContext);
76+
InputFile fileB = addFile(temp, "class B extends A {}", sensorContext);
77+
InputFile fileC = addFile(temp, "class C {}", sensorContext);
78+
79+
long sizeofA = fileA.file().length() + 1;
80+
BatchGenerator generator = new BatchGenerator(
81+
Arrays.asList(fileA, fileB, fileC).iterator(), sizeofA
82+
);
83+
assertThat(generator.hasNext()).isTrue();
84+
assertThat(generator.next()).hasSize(1).contains(fileA);
85+
assertThat(generator.hasNext()).isTrue();
86+
List<InputFile> batchWithMultipleFiles = generator.next();
87+
assertThat(batchWithMultipleFiles).hasSize(2).contains(fileB).contains(fileC);
88+
long batchSize = batchWithMultipleFiles.stream().map(i -> i.file().length()).reduce(0L, Long::sum);
89+
assertThat(batchSize).isLessThanOrEqualTo(sizeofA);
90+
assertThat(generator.hasNext()).isFalse();
91+
assertThat(generator.next()).isEmpty();
92+
93+
long sizeOfAPlusB = fileA.file().length() + fileB.file().length();
94+
generator = new BatchGenerator(
95+
Arrays.asList(fileA, fileB, fileC).iterator(), sizeOfAPlusB
96+
);
97+
assertThat(generator.hasNext()).isTrue();
98+
batchWithMultipleFiles = generator.next();
99+
assertThat(batchWithMultipleFiles).hasSize(2).contains(fileA).contains(fileB);
100+
batchSize = batchWithMultipleFiles.stream().map(i -> i.file().length()).reduce(0L, Long::sum);
101+
assertThat(batchSize).isLessThanOrEqualTo(sizeOfAPlusB);
102+
assertThat(generator.hasNext()).isTrue();
103+
assertThat(generator.next()).hasSize(1).contains(fileC);
104+
assertThat(generator.hasNext()).isFalse();
105+
assertThat(generator.next()).isEmpty();
106+
}
107+
108+
@Test
109+
void batch_generator_includes_file_excluded_from_previous_batch_into_next_batch() throws IOException {
110+
File baseDir = temp.getRoot().getAbsoluteFile();
111+
SensorContextTester sensorContext = SensorContextTester.create(baseDir);
112+
sensorContext.setSettings(new MapSettings());
113+
InputFile fileA = addFile(temp, "class A { public void doSomething() {} }", sensorContext);
114+
InputFile fileB = addFile(temp, "class B extends A {}", sensorContext);
115+
InputFile fileC = addFile(temp, "class C {}", sensorContext);
116+
BatchGenerator generator = new BatchGenerator(
117+
Arrays.asList(fileA, fileC, fileB).iterator(), fileC.file().length()
118+
);
119+
assertThat(generator.hasNext()).isTrue();
120+
assertThat(generator.next()).hasSize(1).contains(fileA);
121+
assertThat(generator.hasNext()).isTrue();
122+
assertThat(generator.next()).hasSize(1).contains(fileC);
123+
assertThat(generator.hasNext()).isTrue();
124+
assertThat(generator.next()).hasSize(1).contains(fileB);
125+
assertThat(generator.hasNext()).isFalse();
126+
assertThat(generator.next()).isEmpty();
127+
}
128+
}

‎java-frontend/src/test/java/org/sonar/java/InputFileUtils.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* SonarQube Java
3+
* Copyright (C) 2012-2025 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the Sonar Source-Available License Version 1, as published by SonarSource SA.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
12+
* See the Sonar Source-Available License for more details.
13+
*
14+
* You should have received a copy of the Sonar Source-Available License
15+
* along with this program; if not, see https://sonarsource.com/license/ssal/
16+
*/
17+
package org.sonar.java;
18+
19+
import com.google.common.io.Files;
20+
import java.io.File;
21+
import java.io.IOException;
22+
import java.nio.charset.StandardCharsets;
23+
import java.util.regex.Matcher;
24+
import java.util.regex.Pattern;
25+
import org.jetbrains.annotations.NotNull;
26+
import org.junit.rules.TemporaryFolder;
27+
import org.sonar.api.batch.fs.InputFile;
28+
import org.sonar.api.batch.sensor.internal.SensorContextTester;
29+
30+
class InputFileUtils {
31+
public static InputFile addFile(TemporaryFolder temp, String code, SensorContextTester context) throws IOException {
32+
Matcher matcher = Pattern.compile("(?:^|\\s)(?:class|interface|enum|record)\\s++(\\w++)").matcher(code);
33+
if (matcher.find()) {
34+
String className = matcher.group(1);
35+
InputFile.Type type = className.endsWith("Test") ? InputFile.Type.TEST : InputFile.Type.MAIN;
36+
File file = temp.newFile(className + ".java").getAbsoluteFile();
37+
return generateInputFile(code, context, file, type);
38+
} else {
39+
File file = temp.newFile("Unnamed.java").getAbsoluteFile();
40+
return generateInputFile(code, context, file, InputFile.Type.MAIN);
41+
}
42+
}
43+
44+
@NotNull
45+
private static InputFile generateInputFile(String code, SensorContextTester context, File file, InputFile.Type type) throws IOException {
46+
Files.asCharSink(file, StandardCharsets.UTF_8).write(code);
47+
InputFile defaultFile = TestUtils.inputFile(context.fileSystem().baseDir().getAbsolutePath(), file, type);
48+
context.fileSystem().add(defaultFile);
49+
return defaultFile;
50+
}
51+
}

0 commit comments

Comments
 (0)