From fcc0579a50f85f9ff0aaa56ded5c9648310a66c0 Mon Sep 17 00:00:00 2001
From: san-zrl <san@zurich.ibm.com>
Date: Fri, 14 Mar 2025 11:19:18 +0100
Subject: [PATCH] fix hanging python scan (issue #138)
Signed-off-by: san-zrl <san@zurich.ibm.com>
---
pom.xml | 42 +++++++--
.../processmanager/ScanProcessManager.java | 85 ++++++++++---------
.../scan/python/PythonScannerService.java | 3 +-
3 files changed, 80 insertions(+), 50 deletions(-)
diff --git a/pom.xml b/pom.xml
index a85dfc542..a5af02d32 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,13 +85,6 @@
<artifactId>quarkus-config-yaml</artifactId>
</dependency>
- <!-- Fixes error with quarkus 3.17 >
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- <version>1.33</version>
- </dependency-->
-
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-hibernate-orm-panache</artifactId>
@@ -135,6 +128,16 @@
<groupId>com.ibm</groupId>
<artifactId>sonar-cryptography-plugin</artifactId>
<version>${sonar.crypto.plugin.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.sonarsource.analyzer-commons</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.woodstox</groupId>
+ <artifactId>woodstox-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.sonarsource.api.plugin</groupId>
@@ -164,6 +167,12 @@
<artifactId>google-java-format</artifactId>
<version>${google-java-format.version}</version>
</dependency>
+
+ <dependency> <!-- Fixes issue #138 -->
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>4.29.0</version> <!-- Use the latest stable version -->
+ </dependency>
</dependencies>
<build>
@@ -352,6 +361,25 @@
<verbose>false</verbose><!-- = ${cyclonedx.verbose} -->
</configuration>
</plugin>
+ <!-- Enable to check dependecy conflicts -->
+ <!--plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>3.0.0-M2</version>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <configuration>
+ <rules>
+ <dependencyConvergence/>
+ </rules>
+ </configuration>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin-->
</plugins>
</build>
<profiles>
diff --git a/src/main/java/com/ibm/usecases/scanning/processmanager/ScanProcessManager.java b/src/main/java/com/ibm/usecases/scanning/processmanager/ScanProcessManager.java
index 60cade032..6728d820e 100644
--- a/src/main/java/com/ibm/usecases/scanning/processmanager/ScanProcessManager.java
+++ b/src/main/java/com/ibm/usecases/scanning/processmanager/ScanProcessManager.java
@@ -57,6 +57,7 @@
import com.ibm.usecases.scanning.services.git.GitService;
import com.ibm.usecases.scanning.services.indexing.JavaIndexService;
import com.ibm.usecases.scanning.services.indexing.ProjectModule;
+import com.ibm.usecases.scanning.services.indexing.PythonIndexService;
import com.ibm.usecases.scanning.services.pkg.MavenPackageFinderService;
import com.ibm.usecases.scanning.services.pkg.SetupPackageFinderService;
import com.ibm.usecases.scanning.services.pkg.TomlPackageFinderService;
@@ -65,6 +66,7 @@
import com.ibm.usecases.scanning.services.resolve.PurlResolver;
import com.ibm.usecases.scanning.services.scan.ScanResultDTO;
import com.ibm.usecases.scanning.services.scan.java.JavaScannerService;
+import com.ibm.usecases.scanning.services.scan.python.PythonScannerService;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
import java.io.File;
@@ -292,11 +294,11 @@ private void handleIndexModulesCommand(@Nonnull IndexModulesCommand command) thr
javaIndexService.index(scanAggregate.getPackageFolder());
this.index.put(Language.JAVA, javaIndex);
// python
- // final PythonIndexService pythonIndexService =
- // new PythonIndexService(this.progressDispatcher, dir);
- // final List<ProjectModule> pythonIndex =
- // pythonIndexService.index(scanAggregate.getPackageFolder());
- // this.index.put(Language.PYTHON, pythonIndex);
+ final PythonIndexService pythonIndexService =
+ new PythonIndexService(this.progressDispatcher, dir);
+ final List<ProjectModule> pythonIndex =
+ pythonIndexService.index(scanAggregate.getPackageFolder());
+ this.index.put(Language.PYTHON, pythonIndex);
// continue with scan
this.commandBus.send(new ScanCommand(command.id()));
} catch (Exception e) {
@@ -372,43 +374,43 @@ private void handleScanCommand(@Nonnull ScanCommand command)
javaScanResultDTO.cbom()));
}
- // // python
- // final PythonScannerService pythonScannerService =
- // new PythonScannerService(
- // this.progressDispatcher,
- // Optional.ofNullable(this.projectDirectory)
- // .orElseThrow(NoProjectDirectoryProvided::new));
- // final ScanResultDTO pythonScanResultDTO =
- // pythonScannerService.scan(
- // gitUrl,
- // scanAggregate.getRevision(),
- // commit,
- // scanAggregate.getPackageFolder().orElse(null),
- // Optional.ofNullable(this.index)
- // .map(i -> i.get(Language.PYTHON))
- // .orElseThrow(NoIndexForProject::new));
- // // update statistics
- // numberOfScannedLine += pythonScanResultDTO.numberOfScannedLine();
- // numberOfScannedFiles += pythonScanResultDTO.numberOfScannedFiles();
+ // python
+ final PythonScannerService pythonScannerService =
+ new PythonScannerService(
+ this.progressDispatcher,
+ Optional.ofNullable(this.projectDirectory)
+ .orElseThrow(NoProjectDirectoryProvided::new));
+ final ScanResultDTO pythonScanResultDTO =
+ pythonScannerService.scan(
+ gitUrl,
+ scanAggregate.getRevision(),
+ commit,
+ scanAggregate.getPackageFolder().orElse(null),
+ Optional.ofNullable(this.index)
+ .map(i -> i.get(Language.PYTHON))
+ .orElseThrow(NoIndexForProject::new));
+ // update statistics
+ numberOfScannedLine += pythonScanResultDTO.numberOfScannedLine();
+ numberOfScannedFiles += pythonScanResultDTO.numberOfScannedFiles();
- // if (pythonScanResultDTO.cbom() != null) {
- // // update statistics
- // if (cbom != null) {
- // cbom.merge(pythonScanResultDTO.cbom());
- // } else {
- // cbom = pythonScanResultDTO.cbom();
- // }
+ if (pythonScanResultDTO.cbom() != null) {
+ // update statistics
+ if (cbom != null) {
+ cbom.merge(pythonScanResultDTO.cbom());
+ } else {
+ cbom = pythonScanResultDTO.cbom();
+ }
- // scanAggregate.reportScanResults(
- // new LanguageScan(
- // Language.PYTHON,
- // new ScanMetadata(
- // pythonScanResultDTO.startTime(),
- // pythonScanResultDTO.endTime(),
- // pythonScanResultDTO.numberOfScannedLine(),
- // pythonScanResultDTO.numberOfScannedFiles()),
- // pythonScanResultDTO.cbom()));
- // }
+ scanAggregate.reportScanResults(
+ new LanguageScan(
+ Language.PYTHON,
+ new ScanMetadata(
+ pythonScanResultDTO.startTime(),
+ pythonScanResultDTO.endTime(),
+ pythonScanResultDTO.numberOfScannedLine(),
+ pythonScanResultDTO.numberOfScannedFiles()),
+ pythonScanResultDTO.cbom()));
+ }
// publish scan finished and save state
scanAggregate.scanFinished();
@@ -435,10 +437,9 @@ private void handleScanCommand(@Nonnull ScanCommand command)
.toString()));
this.progressDispatcher.send(
new ProgressMessage(ProgressMessageType.LABEL, "Finished"));
- } catch (Exception e) {
+ } catch (Exception | NoSuchMethodError e) { // catch NoSuchMethodError: see issue #138
this.progressDispatcher.send(
new ProgressMessage(ProgressMessageType.ERROR, e.getMessage()));
- this.compensate(command.id());
throw e;
} finally {
this.compensate(command.id());
diff --git a/src/main/java/com/ibm/usecases/scanning/services/scan/python/PythonScannerService.java b/src/main/java/com/ibm/usecases/scanning/services/scan/python/PythonScannerService.java
index 19306b87a..ef991c46d 100644
--- a/src/main/java/com/ibm/usecases/scanning/services/scan/python/PythonScannerService.java
+++ b/src/main/java/com/ibm/usecases/scanning/services/scan/python/PythonScannerService.java
@@ -57,6 +57,8 @@ public PythonScannerService(
throws ClientDisconnected {
final PythonCheck visitor = new PythonDetectionCollectionRule(this);
+ LOGGER.info("Start scanning {} python projects", index.size());
+
long scanTimeStart = System.currentTimeMillis();
int counter = 1;
int numberOfScannedLines = 0;
@@ -73,7 +75,6 @@ public PythonScannerService(
ProgressMessageType.LABEL, "Scanning project " + projectStr));
for (InputFile inputFile : project.inputFileList()) {
- LOGGER.info("Scanning file: {}", inputFile.filename());
final PythonScannableFile pythonScannableFile = new PythonScannableFile(inputFile);
final FileInput parsedFile = pythonScannableFile.parse();
final PythonVisitorContext context =