Skip to content

Commit 8396f29

Browse files
san-zrlsan-zrl
authored
fix hanging python scan (issue #138) (#151)
Signed-off-by: san-zrl <[email protected]>
1 parent 08ed497 commit 8396f29

File tree

3 files changed

+80
-50
lines changed

3 files changed

+80
-50
lines changed

‎pom.xml

+35-7
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,6 @@
8585
<artifactId>quarkus-config-yaml</artifactId>
8686
</dependency>
8787

88-
<!-- Fixes error with quarkus 3.17 >
89-
<dependency>
90-
<groupId>org.yaml</groupId>
91-
<artifactId>snakeyaml</artifactId>
92-
<version>1.33</version>
93-
</dependency-->
94-
9588
<dependency>
9689
<groupId>io.quarkus</groupId>
9790
<artifactId>quarkus-hibernate-orm-panache</artifactId>
@@ -135,6 +128,16 @@
135128
<groupId>com.ibm</groupId>
136129
<artifactId>sonar-cryptography-plugin</artifactId>
137130
<version>${sonar.crypto.plugin.version}</version>
131+
<exclusions>
132+
<exclusion>
133+
<groupId>org.sonarsource.analyzer-commons</groupId>
134+
<artifactId>*</artifactId>
135+
</exclusion>
136+
<exclusion>
137+
<groupId>com.fasterxml.woodstox</groupId>
138+
<artifactId>woodstox-core</artifactId>
139+
</exclusion>
140+
</exclusions>
138141
</dependency>
139142
<dependency>
140143
<groupId>org.sonarsource.api.plugin</groupId>
@@ -164,6 +167,12 @@
164167
<artifactId>google-java-format</artifactId>
165168
<version>${google-java-format.version}</version>
166169
</dependency>
170+
171+
<dependency> <!-- Fixes issue #138 -->
172+
<groupId>com.google.protobuf</groupId>
173+
<artifactId>protobuf-java</artifactId>
174+
<version>4.29.0</version> <!-- Use the latest stable version -->
175+
</dependency>
167176
</dependencies>
168177

169178
<build>
@@ -352,6 +361,25 @@
352361
<verbose>false</verbose><!-- = ${cyclonedx.verbose} -->
353362
</configuration>
354363
</plugin>
364+
<!-- Enable to check dependecy conflicts -->
365+
<!--plugin>
366+
<groupId>org.apache.maven.plugins</groupId>
367+
<artifactId>maven-enforcer-plugin</artifactId>
368+
<version>3.0.0-M2</version>
369+
<executions>
370+
<execution>
371+
<id>enforce</id>
372+
<configuration>
373+
<rules>
374+
<dependencyConvergence/>
375+
</rules>
376+
</configuration>
377+
<goals>
378+
<goal>enforce</goal>
379+
</goals>
380+
</execution>
381+
</executions>
382+
</plugin-->
355383
</plugins>
356384
</build>
357385
<profiles>

‎src/main/java/com/ibm/usecases/scanning/processmanager/ScanProcessManager.java

+43-42
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import com.ibm.usecases.scanning.services.git.GitService;
5858
import com.ibm.usecases.scanning.services.indexing.JavaIndexService;
5959
import com.ibm.usecases.scanning.services.indexing.ProjectModule;
60+
import com.ibm.usecases.scanning.services.indexing.PythonIndexService;
6061
import com.ibm.usecases.scanning.services.pkg.MavenPackageFinderService;
6162
import com.ibm.usecases.scanning.services.pkg.SetupPackageFinderService;
6263
import com.ibm.usecases.scanning.services.pkg.TomlPackageFinderService;
@@ -65,6 +66,7 @@
6566
import com.ibm.usecases.scanning.services.resolve.PurlResolver;
6667
import com.ibm.usecases.scanning.services.scan.ScanResultDTO;
6768
import com.ibm.usecases.scanning.services.scan.java.JavaScannerService;
69+
import com.ibm.usecases.scanning.services.scan.python.PythonScannerService;
6870
import jakarta.annotation.Nonnull;
6971
import jakarta.annotation.Nullable;
7072
import java.io.File;
@@ -292,11 +294,11 @@ private void handleIndexModulesCommand(@Nonnull IndexModulesCommand command) thr
292294
javaIndexService.index(scanAggregate.getPackageFolder());
293295
this.index.put(Language.JAVA, javaIndex);
294296
// python
295-
// final PythonIndexService pythonIndexService =
296-
// new PythonIndexService(this.progressDispatcher, dir);
297-
// final List<ProjectModule> pythonIndex =
298-
// pythonIndexService.index(scanAggregate.getPackageFolder());
299-
// this.index.put(Language.PYTHON, pythonIndex);
297+
final PythonIndexService pythonIndexService =
298+
new PythonIndexService(this.progressDispatcher, dir);
299+
final List<ProjectModule> pythonIndex =
300+
pythonIndexService.index(scanAggregate.getPackageFolder());
301+
this.index.put(Language.PYTHON, pythonIndex);
300302
// continue with scan
301303
this.commandBus.send(new ScanCommand(command.id()));
302304
} catch (Exception e) {
@@ -372,43 +374,43 @@ private void handleScanCommand(@Nonnull ScanCommand command)
372374
javaScanResultDTO.cbom()));
373375
}
374376

375-
// // python
376-
// final PythonScannerService pythonScannerService =
377-
// new PythonScannerService(
378-
// this.progressDispatcher,
379-
// Optional.ofNullable(this.projectDirectory)
380-
// .orElseThrow(NoProjectDirectoryProvided::new));
381-
// final ScanResultDTO pythonScanResultDTO =
382-
// pythonScannerService.scan(
383-
// gitUrl,
384-
// scanAggregate.getRevision(),
385-
// commit,
386-
// scanAggregate.getPackageFolder().orElse(null),
387-
// Optional.ofNullable(this.index)
388-
// .map(i -> i.get(Language.PYTHON))
389-
// .orElseThrow(NoIndexForProject::new));
390-
// // update statistics
391-
// numberOfScannedLine += pythonScanResultDTO.numberOfScannedLine();
392-
// numberOfScannedFiles += pythonScanResultDTO.numberOfScannedFiles();
377+
// python
378+
final PythonScannerService pythonScannerService =
379+
new PythonScannerService(
380+
this.progressDispatcher,
381+
Optional.ofNullable(this.projectDirectory)
382+
.orElseThrow(NoProjectDirectoryProvided::new));
383+
final ScanResultDTO pythonScanResultDTO =
384+
pythonScannerService.scan(
385+
gitUrl,
386+
scanAggregate.getRevision(),
387+
commit,
388+
scanAggregate.getPackageFolder().orElse(null),
389+
Optional.ofNullable(this.index)
390+
.map(i -> i.get(Language.PYTHON))
391+
.orElseThrow(NoIndexForProject::new));
392+
// update statistics
393+
numberOfScannedLine += pythonScanResultDTO.numberOfScannedLine();
394+
numberOfScannedFiles += pythonScanResultDTO.numberOfScannedFiles();
393395

394-
// if (pythonScanResultDTO.cbom() != null) {
395-
// // update statistics
396-
// if (cbom != null) {
397-
// cbom.merge(pythonScanResultDTO.cbom());
398-
// } else {
399-
// cbom = pythonScanResultDTO.cbom();
400-
// }
396+
if (pythonScanResultDTO.cbom() != null) {
397+
// update statistics
398+
if (cbom != null) {
399+
cbom.merge(pythonScanResultDTO.cbom());
400+
} else {
401+
cbom = pythonScanResultDTO.cbom();
402+
}
401403

402-
// scanAggregate.reportScanResults(
403-
// new LanguageScan(
404-
// Language.PYTHON,
405-
// new ScanMetadata(
406-
// pythonScanResultDTO.startTime(),
407-
// pythonScanResultDTO.endTime(),
408-
// pythonScanResultDTO.numberOfScannedLine(),
409-
// pythonScanResultDTO.numberOfScannedFiles()),
410-
// pythonScanResultDTO.cbom()));
411-
// }
404+
scanAggregate.reportScanResults(
405+
new LanguageScan(
406+
Language.PYTHON,
407+
new ScanMetadata(
408+
pythonScanResultDTO.startTime(),
409+
pythonScanResultDTO.endTime(),
410+
pythonScanResultDTO.numberOfScannedLine(),
411+
pythonScanResultDTO.numberOfScannedFiles()),
412+
pythonScanResultDTO.cbom()));
413+
}
412414

413415
// publish scan finished and save state
414416
scanAggregate.scanFinished();
@@ -435,10 +437,9 @@ private void handleScanCommand(@Nonnull ScanCommand command)
435437
.toString()));
436438
this.progressDispatcher.send(
437439
new ProgressMessage(ProgressMessageType.LABEL, "Finished"));
438-
} catch (Exception e) {
440+
} catch (Exception | NoSuchMethodError e) { // catch NoSuchMethodError: see issue #138
439441
this.progressDispatcher.send(
440442
new ProgressMessage(ProgressMessageType.ERROR, e.getMessage()));
441-
this.compensate(command.id());
442443
throw e;
443444
} finally {
444445
this.compensate(command.id());

‎src/main/java/com/ibm/usecases/scanning/services/scan/python/PythonScannerService.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ public PythonScannerService(
5757
throws ClientDisconnected {
5858
final PythonCheck visitor = new PythonDetectionCollectionRule(this);
5959

60+
LOGGER.info("Start scanning {} python projects", index.size());
61+
6062
long scanTimeStart = System.currentTimeMillis();
6163
int counter = 1;
6264
int numberOfScannedLines = 0;
@@ -73,7 +75,6 @@ public PythonScannerService(
7375
ProgressMessageType.LABEL, "Scanning project " + projectStr));
7476

7577
for (InputFile inputFile : project.inputFileList()) {
76-
LOGGER.info("Scanning file: {}", inputFile.filename());
7778
final PythonScannableFile pythonScannableFile = new PythonScannableFile(inputFile);
7879
final FileInput parsedFile = pythonScannableFile.parse();
7980
final PythonVisitorContext context =

0 commit comments

Comments
 (0)