From 812e1aca93c45ed4adb84ed661d9f38892be5ea6 Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sat, 16 Nov 2024 16:36:00 -0500 Subject: [PATCH 1/9] add lazy loading to some modules --- .../LazyCodemodLoadingAbstractModule.java | 29 +++++++++++++++++++ .../sarif/appscan/AppScanModule.java | 13 +++++++-- .../providers/sarif/codeql/CodeQLModule.java | 13 +++++++-- .../providers/sarif/pmd/PmdModule.java | 17 +++++++---- .../sarif/semgrep/SemgrepModule.java | 19 +++++++----- .../providers/sonar/SonarModule.java | 13 +++++++-- 6 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 framework/codemodder-base/src/main/java/io/codemodder/LazyCodemodLoadingAbstractModule.java diff --git a/framework/codemodder-base/src/main/java/io/codemodder/LazyCodemodLoadingAbstractModule.java b/framework/codemodder-base/src/main/java/io/codemodder/LazyCodemodLoadingAbstractModule.java new file mode 100644 index 000000000..17944ad34 --- /dev/null +++ b/framework/codemodder-base/src/main/java/io/codemodder/LazyCodemodLoadingAbstractModule.java @@ -0,0 +1,29 @@ +package io.codemodder; + +import com.google.inject.AbstractModule; +import java.util.List; + +/** A module that only loads if it is responsible for a codemod that's being loaded. */ +public abstract class LazyCodemodLoadingAbstractModule extends AbstractModule { + + private final boolean shouldActivate; + + /** Returns true if this module is responsible for any of the given codemods, so it doesn't . */ + protected LazyCodemodLoadingAbstractModule( + final List> codemodTypes) { + this.shouldActivate = codemodTypes.stream().anyMatch(this::isResponsibleFor); + } + + /** Returns true if this module is responsible for any of the given codemods, so it doesn't . */ + protected abstract boolean isResponsibleFor(Class codemod); + + @Override + protected final void configure() { + if (shouldActivate) { + doConfigure(); + } + } + + /** Do the configuration that you would normally do in the configure method. */ + protected abstract void doConfigure(); +} diff --git a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java index 149f48d3b..8ca681f2c 100644 --- a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java +++ b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java @@ -1,7 +1,8 @@ package io.codemodder.providers.sarif.appscan; -import com.google.inject.AbstractModule; import io.codemodder.CodeChanger; +import io.codemodder.Codemod; +import io.codemodder.LazyCodemodLoadingAbstractModule; import io.codemodder.RuleSarif; import java.lang.reflect.Constructor; import java.util.List; @@ -12,19 +13,25 @@ import java.util.stream.Stream; /** Responsible for distributing the SARIFS to AppScan based codemods based on rules. */ -public final class AppScanModule extends AbstractModule { +public final class AppScanModule extends LazyCodemodLoadingAbstractModule { private final List> codemodTypes; private final List allAppScanRuleSarifs; public AppScanModule( final List> codemodTypes, final List sarifs) { + super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.allAppScanRuleSarifs = sarifs; } @Override - protected void configure() { + protected boolean isResponsibleFor(final Class codemod) { + return codemod.getAnnotation(Codemod.class).id().startsWith("appscan:"); + } + + @Override + protected void doConfigure() { final Map map = allAppScanRuleSarifs.stream() .collect(Collectors.toUnmodifiableMap(RuleSarif::getRule, rs -> rs)); diff --git a/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java b/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java index c11f51aa4..74ab62120 100644 --- a/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java +++ b/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java @@ -1,7 +1,8 @@ package io.codemodder.providers.sarif.codeql; -import com.google.inject.AbstractModule; import io.codemodder.CodeChanger; +import io.codemodder.Codemod; +import io.codemodder.LazyCodemodLoadingAbstractModule; import io.codemodder.RuleSarif; import java.lang.reflect.Constructor; import java.util.List; @@ -12,19 +13,25 @@ import java.util.stream.Stream; /** Responsible for distributing the SARIFS to CodeQL based codemods based on rules. */ -public final class CodeQLModule extends AbstractModule { +public final class CodeQLModule extends LazyCodemodLoadingAbstractModule { private final List> codemodTypes; private final List allCodeqlRuleSarifs; CodeQLModule( final List> codemodTypes, final List sarifs) { + super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.allCodeqlRuleSarifs = sarifs; } @Override - protected void configure() { + protected boolean isResponsibleFor(final Class codemod) { + return codemod.getAnnotation(Codemod.class).id().startsWith("codeql:"); + } + + @Override + protected void doConfigure() { // What if there are multiple sarif files with a given rule? // We can safely ignore this case for now. final Map map = diff --git a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java index 51afaa1ca..6d922e972 100644 --- a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java +++ b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java @@ -2,10 +2,7 @@ import com.contrastsecurity.sarif.Result; import com.contrastsecurity.sarif.SarifSchema210; -import com.google.inject.AbstractModule; -import io.codemodder.CodeChanger; -import io.codemodder.LazyLoadingRuleSarif; -import io.codemodder.RuleSarif; +import io.codemodder.*; import io.github.classgraph.*; import java.lang.reflect.Executable; import java.lang.reflect.Parameter; @@ -16,7 +13,7 @@ import org.slf4j.LoggerFactory; /** Responsible for binding PMD-related things. */ -public final class PmdModule extends AbstractModule { +public final class PmdModule extends LazyCodemodLoadingAbstractModule { private final List> codemodTypes; private final Path codeDirectory; @@ -27,6 +24,7 @@ public PmdModule( final Path codeDirectory, final List includedFiles, final List> codemodTypes) { + super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.codeDirectory = Objects.requireNonNull(codeDirectory); this.includedFiles = Objects.requireNonNull(includedFiles); @@ -34,7 +32,14 @@ public PmdModule( } @Override - protected void configure() { + protected boolean isResponsibleFor(final Class codemod) { + Codemod annotation = codemod.getAnnotation(Codemod.class); + String id = annotation.id(); + return id.startsWith("semgrep:") || id.startsWith("pixee:"); + } + + @Override + protected void doConfigure() { Set packagesScanned = new HashSet<>(); List scanTargets = new ArrayList<>(); diff --git a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java index 3f086d991..1e7b82664 100644 --- a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java +++ b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java @@ -2,10 +2,7 @@ import com.contrastsecurity.sarif.Result; import com.contrastsecurity.sarif.SarifSchema210; -import com.google.inject.AbstractModule; -import io.codemodder.CodeChanger; -import io.codemodder.LazyLoadingRuleSarif; -import io.codemodder.RuleSarif; +import io.codemodder.*; import io.github.classgraph.*; import java.io.IOException; import java.io.UncheckedIOException; @@ -20,7 +17,7 @@ import org.slf4j.LoggerFactory; /** Responsible for binding Semgrep-related things. */ -public final class SemgrepModule extends AbstractModule { +public final class SemgrepModule extends LazyCodemodLoadingAbstractModule { private final List> codemodTypes; private final Path codeDirectory; @@ -44,13 +41,14 @@ public SemgrepModule( new DefaultSemgrepRuleFactory()); } - public SemgrepModule( + SemgrepModule( final Path codeDirectory, final List includePatterns, final List excludePatterns, final List> codemodTypes, final List sarifs, final SemgrepRuleFactory semgrepRuleFactory) { + super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.codeDirectory = Objects.requireNonNull(codeDirectory); this.includePatterns = Objects.requireNonNull(includePatterns); @@ -61,7 +59,14 @@ public SemgrepModule( } @Override - protected void configure() { + protected boolean isResponsibleFor(final Class codemod) { + Codemod annotation = codemod.getAnnotation(Codemod.class); + String id = annotation.id(); + return id.startsWith("semgrep:") || id.startsWith("pixee:"); + } + + @Override + protected void doConfigure() { // find all the @ProvidedSemgrepScan annotations and bind them as is Set packagesScanned = new HashSet<>(); diff --git a/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java b/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java index 44c7e6cd5..e7b4edb70 100644 --- a/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java +++ b/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java @@ -1,7 +1,8 @@ package io.codemodder.providers.sonar; -import com.google.inject.AbstractModule; import io.codemodder.CodeChanger; +import io.codemodder.Codemod; +import io.codemodder.LazyCodemodLoadingAbstractModule; import io.codemodder.sonar.model.Hotspot; import io.codemodder.sonar.model.Issue; import io.codemodder.sonar.model.SonarFinding; @@ -14,7 +15,7 @@ import java.util.*; import javax.inject.Inject; -final class SonarModule extends AbstractModule { +final class SonarModule extends LazyCodemodLoadingAbstractModule { private final List> codemodTypes; private final Path repository; @@ -27,6 +28,7 @@ final class SonarModule extends AbstractModule { final Path repository, final List findings, final Class> ruleFindingClass) { + super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.repository = Objects.requireNonNull(repository); this.sonarFindings = findings; @@ -34,7 +36,12 @@ final class SonarModule extends AbstractModule { } @Override - protected void configure() { + protected boolean isResponsibleFor(final Class codemod) { + return codemod.getAnnotation(Codemod.class).id().startsWith("sonar:"); + } + + @Override + protected void doConfigure() { Map> findingsByRuleMap = groupFindingsByRule(sonarFindings); From d764decbf2f0ccc8d9a1d50fbad01fd9a108950c Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sat, 16 Nov 2024 16:41:34 -0500 Subject: [PATCH 2/9] renamed module --- ...tractModule.java => CodemodCheckingAbstractModule.java} | 7 +++---- .../src/main/kotlin/io.codemodder.maven-publish.gradle.kts | 2 +- .../codemodder/providers/sarif/appscan/AppScanModule.java | 4 ++-- .../io/codemodder/providers/sarif/codeql/CodeQLModule.java | 4 ++-- .../java/io/codemodder/providers/sarif/pmd/PmdModule.java | 2 +- .../codemodder/providers/sarif/semgrep/SemgrepModule.java | 2 +- .../java/io/codemodder/providers/sonar/SonarModule.java | 4 ++-- 7 files changed, 12 insertions(+), 13 deletions(-) rename framework/codemodder-base/src/main/java/io/codemodder/{LazyCodemodLoadingAbstractModule.java => CodemodCheckingAbstractModule.java} (72%) diff --git a/framework/codemodder-base/src/main/java/io/codemodder/LazyCodemodLoadingAbstractModule.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java similarity index 72% rename from framework/codemodder-base/src/main/java/io/codemodder/LazyCodemodLoadingAbstractModule.java rename to framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java index 17944ad34..1c3b7bfb1 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/LazyCodemodLoadingAbstractModule.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java @@ -3,14 +3,13 @@ import com.google.inject.AbstractModule; import java.util.List; -/** A module that only loads if it is responsible for a codemod that's being loaded. */ -public abstract class LazyCodemodLoadingAbstractModule extends AbstractModule { +/** A module that only configures if it is responsible for a codemod that's being loaded. */ +public abstract class CodemodCheckingAbstractModule extends AbstractModule { private final boolean shouldActivate; /** Returns true if this module is responsible for any of the given codemods, so it doesn't . */ - protected LazyCodemodLoadingAbstractModule( - final List> codemodTypes) { + protected CodemodCheckingAbstractModule(final List> codemodTypes) { this.shouldActivate = codemodTypes.stream().anyMatch(this::isResponsibleFor); } diff --git a/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts b/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts index 39bd2d9d1..dea87013e 100644 --- a/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts +++ b/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts @@ -26,7 +26,7 @@ signing { val signingPassword: String? by project useInMemoryPgpKeys(signingKey, signingPassword) } - sign(extensions.getByType().publications.getByName(publicationName)) +// sign(extensions.getByType().publications.getByName(publicationName)) } publishing { diff --git a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java index 8ca681f2c..7ad828c92 100644 --- a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java +++ b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java @@ -2,7 +2,7 @@ import io.codemodder.CodeChanger; import io.codemodder.Codemod; -import io.codemodder.LazyCodemodLoadingAbstractModule; +import io.codemodder.CodemodCheckingAbstractModule; import io.codemodder.RuleSarif; import java.lang.reflect.Constructor; import java.util.List; @@ -13,7 +13,7 @@ import java.util.stream.Stream; /** Responsible for distributing the SARIFS to AppScan based codemods based on rules. */ -public final class AppScanModule extends LazyCodemodLoadingAbstractModule { +public final class AppScanModule extends CodemodCheckingAbstractModule { private final List> codemodTypes; private final List allAppScanRuleSarifs; diff --git a/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java b/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java index 74ab62120..d51223cfe 100644 --- a/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java +++ b/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java @@ -2,7 +2,7 @@ import io.codemodder.CodeChanger; import io.codemodder.Codemod; -import io.codemodder.LazyCodemodLoadingAbstractModule; +import io.codemodder.CodemodCheckingAbstractModule; import io.codemodder.RuleSarif; import java.lang.reflect.Constructor; import java.util.List; @@ -13,7 +13,7 @@ import java.util.stream.Stream; /** Responsible for distributing the SARIFS to CodeQL based codemods based on rules. */ -public final class CodeQLModule extends LazyCodemodLoadingAbstractModule { +public final class CodeQLModule extends CodemodCheckingAbstractModule { private final List> codemodTypes; private final List allCodeqlRuleSarifs; diff --git a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java index 6d922e972..4666871df 100644 --- a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java +++ b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java @@ -13,7 +13,7 @@ import org.slf4j.LoggerFactory; /** Responsible for binding PMD-related things. */ -public final class PmdModule extends LazyCodemodLoadingAbstractModule { +public final class PmdModule extends CodemodCheckingAbstractModule { private final List> codemodTypes; private final Path codeDirectory; diff --git a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java index 1e7b82664..0aa559136 100644 --- a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java +++ b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java @@ -17,7 +17,7 @@ import org.slf4j.LoggerFactory; /** Responsible for binding Semgrep-related things. */ -public final class SemgrepModule extends LazyCodemodLoadingAbstractModule { +public final class SemgrepModule extends CodemodCheckingAbstractModule { private final List> codemodTypes; private final Path codeDirectory; diff --git a/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java b/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java index e7b4edb70..fd8b91c33 100644 --- a/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java +++ b/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java @@ -2,7 +2,7 @@ import io.codemodder.CodeChanger; import io.codemodder.Codemod; -import io.codemodder.LazyCodemodLoadingAbstractModule; +import io.codemodder.CodemodCheckingAbstractModule; import io.codemodder.sonar.model.Hotspot; import io.codemodder.sonar.model.Issue; import io.codemodder.sonar.model.SonarFinding; @@ -15,7 +15,7 @@ import java.util.*; import javax.inject.Inject; -final class SonarModule extends LazyCodemodLoadingAbstractModule { +final class SonarModule extends CodemodCheckingAbstractModule { private final List> codemodTypes; private final Path repository; From 26b851caa78a387557c89ec24b6bf8ca237acf2a Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sun, 17 Nov 2024 17:02:32 -0500 Subject: [PATCH 3/9] cached location lookup data --- .../src/main/java/io/codemodder/CLI.java | 8 +++ .../CodemodCheckingAbstractModule.java | 6 ++ .../java/io/codemodder/CodemodLoader.java | 9 +++ .../io/codemodder/DefaultSarifParser.java | 38 ++++++++----- .../io/codemodder/LoggingConfigurator.java | 4 +- .../io.codemodder.maven-publish.gradle.kts | 2 +- .../sarif/appscan/AppScanRuleSarif.java | 45 ++++----------- .../appscan/AppScanRuleSarifFactory.java | 16 +++++- .../appscan/AppScanSarifLocationData.java | 55 +++++++++++++++++++ .../sarif/appscan/AppScanModuleTest.java | 2 +- 10 files changed, 133 insertions(+), 52 deletions(-) create mode 100644 plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CLI.java b/framework/codemodder-base/src/main/java/io/codemodder/CLI.java index f0297d0dd..c5b6bf5f5 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/CLI.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/CLI.java @@ -366,10 +366,15 @@ public Integer call() throws IOException { log.debug("excluding paths: {}", pathExcludes); // get all files that match + log.debug("Listing source directories"); List sourceDirectories = sourceDirectoryLister.listJavaSourceDirectories(List.of(projectDirectory)); + + log.debug("Listing files"); List filePaths = fileFinder.findFiles(projectPath, includesExcludes); + log.debug("Creating codemod regulator"); + // get codemod includes/excludes final CodemodRegulator regulator; if (codemodIncludes != null && codemodExcludes != null) { @@ -386,10 +391,13 @@ public Integer call() throws IOException { } // create the loader + log.debug("Loading input files"); CodeDirectory codeDirectory = new DefaultCodeDirectory(projectPath); List sarifFiles = convertToPaths(sarifs); List sonarIssuesJsonFiles = convertToPaths(sonarIssuesJsonFilePaths); List sonarHotspotJsonFiles = convertToPaths(sonarHotspotsJsonFilePaths); + + log.debug("Parsing SARIFs"); Map> pathSarifMap = SarifParser.create().parseIntoMap(sarifFiles, codeDirectory); List codemodParameters = diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java index 1c3b7bfb1..942dea82d 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java @@ -2,6 +2,8 @@ import com.google.inject.AbstractModule; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** A module that only configures if it is responsible for a codemod that's being loaded. */ public abstract class CodemodCheckingAbstractModule extends AbstractModule { @@ -19,10 +21,14 @@ protected CodemodCheckingAbstractModule(final List> @Override protected final void configure() { if (shouldActivate) { + log.info("Configuring module {}", this.getClass().getSimpleName()); doConfigure(); + log.info("Done configuration {}", this.getClass().getSimpleName()); } } /** Do the configuration that you would normally do in the configure method. */ protected abstract void doConfigure(); + + private static final Logger log = LoggerFactory.getLogger(CodemodCheckingAbstractModule.class); } diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java index 44ce46684..a4aa47622 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java @@ -13,6 +13,8 @@ import java.util.regex.Pattern; import javax.inject.Inject; import org.jetbrains.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** This type is responsible for loading codemods and the surrounding subsystem. */ public final class CodemodLoader { @@ -33,6 +35,8 @@ public CodemodLoader( final Path defectDojoFindingsJsonFile, final Path contrastVulnerabilitiesXmlFilePath) { + log.info("Loading providers"); + // get all the providers ready for dependency injection & codemod instantiation final List providers = ServiceLoader.load(CodemodProvider.class).stream() @@ -106,6 +110,7 @@ public CodemodLoader( wantsSarif.stream() .flatMap(toolName -> ruleSarifByTool.getOrDefault(toolName, List.of()).stream()) .toList(); + log.info("Loading modules from provider: {}", provider.getClass().getSimpleName()); final Set modules = provider.getModules( repositoryDir, @@ -125,7 +130,9 @@ public CodemodLoader( final List codemods = new ArrayList<>(); // validate and instantiate the codemods + log.info("Instantiating codemods"); final Injector injector = Guice.createInjector(allModules); + log.info("Codemods instantiated"); final Set codemodIds = new HashSet<>(); for (final Class type : orderedCodemodTypes) { final Codemod codemodAnnotation = type.getAnnotation(Codemod.class); @@ -164,6 +171,8 @@ static boolean isValidCodemodId(final String codemodId) { return codemodIdPattern.matcher(codemodId).matches(); } + private static final Logger log = LoggerFactory.getLogger(CodemodLoader.class); + private static final Pattern codemodIdPattern = Pattern.compile("^([A-Za-z0-9]+):(([A-Za-z0-9]+)/)+([A-Za-z0-9\\-\\.]+)$"); } diff --git a/framework/codemodder-base/src/main/java/io/codemodder/DefaultSarifParser.java b/framework/codemodder-base/src/main/java/io/codemodder/DefaultSarifParser.java index d1a6ba370..0c7b5b4fc 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/DefaultSarifParser.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/DefaultSarifParser.java @@ -5,6 +5,7 @@ import com.contrastsecurity.sarif.SarifSchema210; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; @@ -17,10 +18,14 @@ final class DefaultSarifParser implements SarifParser { private Optional readSarifFile(final Path sarifFile) { try { - return Optional.of( - new ObjectMapper().readValue(Files.newInputStream(sarifFile), SarifSchema210.class)); + log.debug("Reading input file: {}", sarifFile); + InputStream stream = Files.newInputStream(sarifFile); + log.debug("Parsing to SARIF input files"); + SarifSchema210 sarif = new ObjectMapper().readValue(stream, SarifSchema210.class); + log.debug("Parsed SARIF input files"); + return Optional.of(sarif); } catch (final IOException e) { - LOG.error("Problem deserializing SARIF file: {}", sarifFile, e); + log.error("Problem deserializing SARIF file: {}", sarifFile, e); return Optional.empty(); } } @@ -33,6 +38,7 @@ private Optional> tryToBuild( final CodeDirectory codeDirectory, final List factories) { for (final var factory : factories) { + log.debug("Building SARIF: {}", factory.getClass().getSimpleName()); final var maybeRuleSarif = factory.build(toolName, rule.ruleId, rule.messageText, sarif, codeDirectory); if (maybeRuleSarif.isPresent()) { @@ -40,7 +46,7 @@ private Optional> tryToBuild( } } - LOG.info("Found SARIF from unsupported tool: {}", toolName); + log.info("Found SARIF from unsupported tool: {}", toolName); return Optional.empty(); } @@ -60,7 +66,7 @@ private RuleDescriptor extractRuleId(final Result result, final Run run) { if (maybeRule.isPresent()) { return maybeRule.get(); } else { - LOG.info("Could not find rule id for result."); + log.info("Could not find rule id for result."); return null; } } @@ -72,10 +78,12 @@ private Stream> fromSarif( final Run run, final SarifSchema210 sarif, final CodeDirectory codeDirectory) { // driver name final var toolName = run.getTool().getDriver().getName(); + log.debug("Loading SARIF rule factories"); final List factories = ServiceLoader.load(RuleSarifFactory.class).stream() .map(ServiceLoader.Provider::get) .toList(); + log.debug("Done loading SARIF rule factories"); final var runResults = run.getResults(); final var allResults = runResults != null @@ -105,16 +113,18 @@ public Map> parseIntoMap( .flatMap( sarif -> sarif.getRuns().stream().flatMap(run -> fromSarif(run, sarif, codeDirectory))) .forEach( - p -> - map.merge( - p.getKey(), - new ArrayList<>(Collections.singletonList(p.getValue())), - (l1, l2) -> { - l1.add(l2.get(0)); - return l1; - })); + p -> { + log.debug("Merging SARIF results"); + map.merge( + p.getKey(), + new ArrayList<>(Collections.singletonList(p.getValue())), + (l1, l2) -> { + l1.add(l2.get(0)); + return l1; + }); + }); return map; } - private static final Logger LOG = LoggerFactory.getLogger(DefaultSarifParser.class); + private static final Logger log = LoggerFactory.getLogger(DefaultSarifParser.class); } diff --git a/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java b/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java index 506757dec..42876df10 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java @@ -24,7 +24,9 @@ public ExecutionStatus configure(final LoggerContext lc) { ca.setContext(lc); ca.setName(APPENDER_NAME); PatternLayoutEncoder patternLayoutEncoder = new PatternLayoutEncoder(); - patternLayoutEncoder.setPattern("%m%n"); + // add the timestamp + // patternLayoutEncoder.setPattern("%m%n"); + patternLayoutEncoder.setPattern("%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"); patternLayoutEncoder.setContext(lc); patternLayoutEncoder.setCharset(StandardCharsets.UTF_8); patternLayoutEncoder.start(); diff --git a/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts b/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts index dea87013e..39bd2d9d1 100644 --- a/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts +++ b/gradle/build-plugins/src/main/kotlin/io.codemodder.maven-publish.gradle.kts @@ -26,7 +26,7 @@ signing { val signingPassword: String? by project useInMemoryPgpKeys(signingKey, signingPassword) } -// sign(extensions.getByType().publications.getByName(publicationName)) + sign(extensions.getByType().publications.getByName(publicationName)) } publishing { diff --git a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarif.java b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarif.java index 93fd6ebab..016c0d047 100644 --- a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarif.java +++ b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarif.java @@ -1,12 +1,11 @@ package io.codemodder.providers.sarif.appscan; import com.contrastsecurity.sarif.*; -import io.codemodder.CodeDirectory; import io.codemodder.RuleSarif; -import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** A {@link RuleSarif} for AppScan results. */ final class AppScanRuleSarif implements RuleSarif { @@ -14,45 +13,20 @@ final class AppScanRuleSarif implements RuleSarif { private final SarifSchema210 sarif; private final String messageText; private final Map> resultsCache; - private final List locations; - - /** A map of a AppScan SARIF "location" URIs mapped to their respective file paths. */ - private final Map> artifactLocationIndices; + private final AppScanSarifLocationData sarifLocationData; /** * Creates an {@link AppScanRuleSarif} that has already done the work of mapping AppScan SARIF * locations, which are strange combinations of class name and file path, into predictable paths. */ AppScanRuleSarif( - final String messageText, final SarifSchema210 sarif, final CodeDirectory codeDirectory) { + final String messageText, + final SarifSchema210 sarif, + final AppScanSarifLocationData sarifLocationData) { this.sarif = Objects.requireNonNull(sarif); this.messageText = Objects.requireNonNull(messageText); this.resultsCache = new HashMap<>(); - this.locations = - sarif.getRuns().get(0).getArtifacts().stream() - .map(Artifact::getLocation) - .map(ArtifactLocation::getUri) - .map(u -> u.substring(8)) // trim the file:/// of all results - .toList(); - Map> artifactLocationIndicesMap = new HashMap<>(); - - for (int i = 0; i < locations.size(); i++) { - final Integer index = i; - String path = locations.get(i); - path = path.replace('\\', '/'); - // we have a real but partial path, now we have to find it in the repository - Optional existingRealPath; - try { - existingRealPath = codeDirectory.findFilesWithTrailingPath(path); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - // add to the map if we found a matching file - existingRealPath.ifPresent( - p -> artifactLocationIndicesMap.computeIfAbsent(p, k -> new HashSet<>()).add(index)); - } - this.artifactLocationIndices = Map.copyOf(artifactLocationIndicesMap); + this.sarifLocationData = Objects.requireNonNull(sarifLocationData); } @Override @@ -71,6 +45,9 @@ public List getRegionsFromResultsByRule(final Path path) { */ @Override public List getResultsByLocationPath(final Path path) { + + Map> artifactLocationIndices = + sarifLocationData.getArtifactLocationIndices(); return resultsCache.computeIfAbsent( path, p -> @@ -117,4 +94,6 @@ public String getRule() { } static final String toolName = "HCL AppScan Static Analyzer"; + + private static final Logger log = LoggerFactory.getLogger(AppScanRuleSarif.class); } diff --git a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarifFactory.java b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarifFactory.java index 3f733a786..8710d244c 100644 --- a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarifFactory.java +++ b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarifFactory.java @@ -4,11 +4,18 @@ import io.codemodder.CodeDirectory; import io.codemodder.RuleSarif; import io.codemodder.RuleSarifFactory; -import java.util.Optional; +import java.util.*; /** A factory for building {@link AppScanRuleSarif}s. */ public final class AppScanRuleSarifFactory implements RuleSarifFactory { + /** A map of a AppScan SARIF "location" URIs mapped to their respective file paths. */ + private final Map sarifLocationDataCache; + + public AppScanRuleSarifFactory() { + this.sarifLocationDataCache = new HashMap<>(); + } + @Override public Optional build( final String toolName, @@ -17,7 +24,12 @@ public Optional build( final SarifSchema210 sarif, final CodeDirectory codeDirectory) { if (AppScanRuleSarif.toolName.equals(toolName)) { - return Optional.of(new AppScanRuleSarif(messageText, sarif, codeDirectory)); + AppScanSarifLocationData sarifLocationData = sarifLocationDataCache.get(sarif); + if (sarifLocationData == null) { + sarifLocationData = new AppScanSarifLocationData(sarif, codeDirectory); + sarifLocationDataCache.put(sarif, sarifLocationData); + } + return Optional.of(new AppScanRuleSarif(messageText, sarif, sarifLocationData)); } return Optional.empty(); } diff --git a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java new file mode 100644 index 000000000..0145e7369 --- /dev/null +++ b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java @@ -0,0 +1,55 @@ +package io.codemodder.providers.sarif.appscan; + +import com.contrastsecurity.sarif.Artifact; +import com.contrastsecurity.sarif.ArtifactLocation; +import com.contrastsecurity.sarif.SarifSchema210; +import io.codemodder.CodeDirectory; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Holds the locations of the artifacts in the SARIF file. */ +final class AppScanSarifLocationData { + + private final Map> artifactLocationIndices; + + AppScanSarifLocationData(final SarifSchema210 sarif, final CodeDirectory codeDirectory) { + log.debug("Cleaning locations"); + List locations = + sarif.getRuns().get(0).getArtifacts().stream() + .map(Artifact::getLocation) + .map(ArtifactLocation::getUri) + .map(u -> u.substring(8)) // trim the file:/// of all results + .toList(); + + Map> artifactLocationIndicesMap = new HashMap<>(); + + log.info("Calculating locations in project dir"); + for (int i = 0; i < locations.size(); i++) { + final Integer index = i; + String path = locations.get(i); + path = path.replace('\\', '/'); + // we have a real but partial path, now we have to find it in the repository + Optional existingRealPath; + try { + existingRealPath = codeDirectory.findFilesWithTrailingPath(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + // add to the map if we found a matching file + existingRealPath.ifPresent( + p -> artifactLocationIndicesMap.computeIfAbsent(p, k -> new HashSet<>()).add(index)); + } + log.info("Done calculating locations"); + this.artifactLocationIndices = Map.copyOf(artifactLocationIndicesMap); + } + + Map> getArtifactLocationIndices() { + return artifactLocationIndices; + } + + private static final Logger log = LoggerFactory.getLogger(AppScanSarifLocationData.class); +} diff --git a/plugins/codemodder-plugin-appscan/src/test/java/io/codemodder/providers/sarif/appscan/AppScanModuleTest.java b/plugins/codemodder-plugin-appscan/src/test/java/io/codemodder/providers/sarif/appscan/AppScanModuleTest.java index 90a92285b..029eeed58 100644 --- a/plugins/codemodder-plugin-appscan/src/test/java/io/codemodder/providers/sarif/appscan/AppScanModuleTest.java +++ b/plugins/codemodder-plugin-appscan/src/test/java/io/codemodder/providers/sarif/appscan/AppScanModuleTest.java @@ -21,7 +21,7 @@ final class AppScanModuleTest { @Codemod( - id = "appscan-test:java/finds-stuff", + id = "appscan:java/finds-stuff", importance = Importance.LOW, reviewGuidance = ReviewGuidance.MERGE_AFTER_CURSORY_REVIEW) static class AppScanSarifTestCodemod implements CodeChanger { From ebe71d9d8f5c10aad0b45bd662c49dffe913d2d9 Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sun, 17 Nov 2024 17:04:26 -0500 Subject: [PATCH 4/9] move to right level --- .../java/io/codemodder/CodemodCheckingAbstractModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java index 942dea82d..e9a721b57 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java @@ -21,9 +21,9 @@ protected CodemodCheckingAbstractModule(final List> @Override protected final void configure() { if (shouldActivate) { - log.info("Configuring module {}", this.getClass().getSimpleName()); + log.debug("Configuring module {}", this.getClass().getSimpleName()); doConfigure(); - log.info("Done configuration {}", this.getClass().getSimpleName()); + log.debug("Done configuration {}", this.getClass().getSimpleName()); } } From b7c33b50bf7996563c8d75b114e4ffad49a0ca53 Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sun, 17 Nov 2024 17:06:15 -0500 Subject: [PATCH 5/9] moved more logs to the right level --- .../src/main/java/io/codemodder/CodemodLoader.java | 8 ++++---- .../providers/sarif/appscan/AppScanSarifLocationData.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java index a4aa47622..75640b1b6 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java @@ -35,7 +35,7 @@ public CodemodLoader( final Path defectDojoFindingsJsonFile, final Path contrastVulnerabilitiesXmlFilePath) { - log.info("Loading providers"); + log.debug("Loading providers"); // get all the providers ready for dependency injection & codemod instantiation final List providers = @@ -110,7 +110,7 @@ public CodemodLoader( wantsSarif.stream() .flatMap(toolName -> ruleSarifByTool.getOrDefault(toolName, List.of()).stream()) .toList(); - log.info("Loading modules from provider: {}", provider.getClass().getSimpleName()); + log.debug("Loading modules from provider: {}", provider.getClass().getSimpleName()); final Set modules = provider.getModules( repositoryDir, @@ -130,9 +130,9 @@ public CodemodLoader( final List codemods = new ArrayList<>(); // validate and instantiate the codemods - log.info("Instantiating codemods"); + log.debug("Instantiating codemods"); final Injector injector = Guice.createInjector(allModules); - log.info("Codemods instantiated"); + log.debug("Codemods instantiated"); final Set codemodIds = new HashSet<>(); for (final Class type : orderedCodemodTypes) { final Codemod codemodAnnotation = type.getAnnotation(Codemod.class); diff --git a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java index 0145e7369..ca2acc11a 100644 --- a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java +++ b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java @@ -27,7 +27,7 @@ final class AppScanSarifLocationData { Map> artifactLocationIndicesMap = new HashMap<>(); - log.info("Calculating locations in project dir"); + log.debug("Calculating locations in project dir"); for (int i = 0; i < locations.size(); i++) { final Integer index = i; String path = locations.get(i); @@ -43,7 +43,7 @@ final class AppScanSarifLocationData { existingRealPath.ifPresent( p -> artifactLocationIndicesMap.computeIfAbsent(p, k -> new HashSet<>()).add(index)); } - log.info("Done calculating locations"); + log.debug("Done calculating locations"); this.artifactLocationIndices = Map.copyOf(artifactLocationIndicesMap); } From 7380b959606289ff7cbfba4a7496dcb4aa526f2a Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sun, 17 Nov 2024 17:07:52 -0500 Subject: [PATCH 6/9] restore log format --- .../src/main/java/io/codemodder/LoggingConfigurator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java b/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java index 42876df10..506757dec 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/LoggingConfigurator.java @@ -24,9 +24,7 @@ public ExecutionStatus configure(final LoggerContext lc) { ca.setContext(lc); ca.setName(APPENDER_NAME); PatternLayoutEncoder patternLayoutEncoder = new PatternLayoutEncoder(); - // add the timestamp - // patternLayoutEncoder.setPattern("%m%n"); - patternLayoutEncoder.setPattern("%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"); + patternLayoutEncoder.setPattern("%m%n"); patternLayoutEncoder.setContext(lc); patternLayoutEncoder.setCharset(StandardCharsets.UTF_8); patternLayoutEncoder.start(); From 43e07472603deba344666998f4f6432fbad4da16 Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sun, 17 Nov 2024 17:17:24 -0500 Subject: [PATCH 7/9] allow codemodder namespace to activate semgrep module --- .../io/codemodder/providers/sarif/semgrep/SemgrepModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java index 0aa559136..aa0ac5280 100644 --- a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java +++ b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java @@ -62,7 +62,7 @@ public SemgrepModule( protected boolean isResponsibleFor(final Class codemod) { Codemod annotation = codemod.getAnnotation(Codemod.class); String id = annotation.id(); - return id.startsWith("semgrep:") || id.startsWith("pixee:"); + return id.startsWith("semgrep:") || id.startsWith("pixee:") || id.startsWith("codemodder:"); } @Override From b4743682b6f84859fefd82dbc9f48b53049581d4 Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sun, 17 Nov 2024 17:36:10 -0500 Subject: [PATCH 8/9] activate pmd module on pmd namespaced codemods --- .../main/java/io/codemodder/providers/sarif/pmd/PmdModule.java | 2 +- .../java/io/codemodder/providers/sarif/pmd/PmdModuleTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java index 4666871df..32cc307b2 100644 --- a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java +++ b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java @@ -35,7 +35,7 @@ public PmdModule( protected boolean isResponsibleFor(final Class codemod) { Codemod annotation = codemod.getAnnotation(Codemod.class); String id = annotation.id(); - return id.startsWith("semgrep:") || id.startsWith("pixee:"); + return id.startsWith("semgrep:") || id.startsWith("pixee:") || id.startsWith("pmd:"); } @Override diff --git a/plugins/codemodder-plugin-pmd/src/test/java/io/codemodder/providers/sarif/pmd/PmdModuleTest.java b/plugins/codemodder-plugin-pmd/src/test/java/io/codemodder/providers/sarif/pmd/PmdModuleTest.java index 8d635de2e..97ff63c3c 100644 --- a/plugins/codemodder-plugin-pmd/src/test/java/io/codemodder/providers/sarif/pmd/PmdModuleTest.java +++ b/plugins/codemodder-plugin-pmd/src/test/java/io/codemodder/providers/sarif/pmd/PmdModuleTest.java @@ -46,7 +46,7 @@ public abstract class MultipleDeclarations { } @Codemod( - id = "pmd-test:java/my-pmd-codemod", + id = "pmd:java/my-pmd-codemod", importance = Importance.HIGH, reviewGuidance = ReviewGuidance.MERGE_AFTER_CURSORY_REVIEW) static class UsesPmdCodemod extends SarifPluginJavaParserChanger { From b4c80205d175f3735d6915c0cae702211e6f9e14 Mon Sep 17 00:00:00 2001 From: Arshan Dabirsiaghi Date: Sun, 17 Nov 2024 18:03:39 -0500 Subject: [PATCH 9/9] removing unnecessary changes --- .../CodemodCheckingAbstractModule.java | 34 ------------------- .../sarif/appscan/AppScanModule.java | 13 ++----- .../providers/sarif/codeql/CodeQLModule.java | 13 ++----- .../providers/sarif/pmd/PmdModule.java | 18 +++++----- .../sarif/semgrep/SemgrepModule.java | 21 ++++++------ .../providers/sonar/SonarModule.java | 13 ++----- 6 files changed, 28 insertions(+), 84 deletions(-) delete mode 100644 framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java deleted file mode 100644 index e9a721b57..000000000 --- a/framework/codemodder-base/src/main/java/io/codemodder/CodemodCheckingAbstractModule.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.codemodder; - -import com.google.inject.AbstractModule; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** A module that only configures if it is responsible for a codemod that's being loaded. */ -public abstract class CodemodCheckingAbstractModule extends AbstractModule { - - private final boolean shouldActivate; - - /** Returns true if this module is responsible for any of the given codemods, so it doesn't . */ - protected CodemodCheckingAbstractModule(final List> codemodTypes) { - this.shouldActivate = codemodTypes.stream().anyMatch(this::isResponsibleFor); - } - - /** Returns true if this module is responsible for any of the given codemods, so it doesn't . */ - protected abstract boolean isResponsibleFor(Class codemod); - - @Override - protected final void configure() { - if (shouldActivate) { - log.debug("Configuring module {}", this.getClass().getSimpleName()); - doConfigure(); - log.debug("Done configuration {}", this.getClass().getSimpleName()); - } - } - - /** Do the configuration that you would normally do in the configure method. */ - protected abstract void doConfigure(); - - private static final Logger log = LoggerFactory.getLogger(CodemodCheckingAbstractModule.class); -} diff --git a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java index 7ad828c92..149f48d3b 100644 --- a/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java +++ b/plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanModule.java @@ -1,8 +1,7 @@ package io.codemodder.providers.sarif.appscan; +import com.google.inject.AbstractModule; import io.codemodder.CodeChanger; -import io.codemodder.Codemod; -import io.codemodder.CodemodCheckingAbstractModule; import io.codemodder.RuleSarif; import java.lang.reflect.Constructor; import java.util.List; @@ -13,25 +12,19 @@ import java.util.stream.Stream; /** Responsible for distributing the SARIFS to AppScan based codemods based on rules. */ -public final class AppScanModule extends CodemodCheckingAbstractModule { +public final class AppScanModule extends AbstractModule { private final List> codemodTypes; private final List allAppScanRuleSarifs; public AppScanModule( final List> codemodTypes, final List sarifs) { - super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.allAppScanRuleSarifs = sarifs; } @Override - protected boolean isResponsibleFor(final Class codemod) { - return codemod.getAnnotation(Codemod.class).id().startsWith("appscan:"); - } - - @Override - protected void doConfigure() { + protected void configure() { final Map map = allAppScanRuleSarifs.stream() .collect(Collectors.toUnmodifiableMap(RuleSarif::getRule, rs -> rs)); diff --git a/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java b/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java index d51223cfe..c11f51aa4 100644 --- a/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java +++ b/plugins/codemodder-plugin-codeql/src/main/java/io/codemodder/providers/sarif/codeql/CodeQLModule.java @@ -1,8 +1,7 @@ package io.codemodder.providers.sarif.codeql; +import com.google.inject.AbstractModule; import io.codemodder.CodeChanger; -import io.codemodder.Codemod; -import io.codemodder.CodemodCheckingAbstractModule; import io.codemodder.RuleSarif; import java.lang.reflect.Constructor; import java.util.List; @@ -13,25 +12,19 @@ import java.util.stream.Stream; /** Responsible for distributing the SARIFS to CodeQL based codemods based on rules. */ -public final class CodeQLModule extends CodemodCheckingAbstractModule { +public final class CodeQLModule extends AbstractModule { private final List> codemodTypes; private final List allCodeqlRuleSarifs; CodeQLModule( final List> codemodTypes, final List sarifs) { - super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.allCodeqlRuleSarifs = sarifs; } @Override - protected boolean isResponsibleFor(final Class codemod) { - return codemod.getAnnotation(Codemod.class).id().startsWith("codeql:"); - } - - @Override - protected void doConfigure() { + protected void configure() { // What if there are multiple sarif files with a given rule? // We can safely ignore this case for now. final Map map = diff --git a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java index 32cc307b2..45ab4d437 100644 --- a/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java +++ b/plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java @@ -2,6 +2,7 @@ import com.contrastsecurity.sarif.Result; import com.contrastsecurity.sarif.SarifSchema210; +import com.google.inject.AbstractModule; import io.codemodder.*; import io.github.classgraph.*; import java.lang.reflect.Executable; @@ -13,7 +14,7 @@ import org.slf4j.LoggerFactory; /** Responsible for binding PMD-related things. */ -public final class PmdModule extends CodemodCheckingAbstractModule { +public final class PmdModule extends AbstractModule { private final List> codemodTypes; private final Path codeDirectory; @@ -24,7 +25,6 @@ public PmdModule( final Path codeDirectory, final List includedFiles, final List> codemodTypes) { - super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.codeDirectory = Objects.requireNonNull(codeDirectory); this.includedFiles = Objects.requireNonNull(includedFiles); @@ -32,14 +32,7 @@ public PmdModule( } @Override - protected boolean isResponsibleFor(final Class codemod) { - Codemod annotation = codemod.getAnnotation(Codemod.class); - String id = annotation.id(); - return id.startsWith("semgrep:") || id.startsWith("pixee:") || id.startsWith("pmd:"); - } - - @Override - protected void doConfigure() { + protected void configure() { Set packagesScanned = new HashSet<>(); List scanTargets = new ArrayList<>(); @@ -88,6 +81,11 @@ protected void doConfigure() { } } + if (scanTargets.isEmpty()) { + LOG.trace("No @PmdScan annotations found, skipping"); + return; + } + SarifSchema210 allRulesBatchedRun = pmdRunner.run( scanTargets.stream().map(PmdScanTarget::pmdScan).map(PmdScan::ruleId).toList(), diff --git a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java index aa0ac5280..9a5b8fefd 100644 --- a/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java +++ b/plugins/codemodder-plugin-semgrep/src/main/java/io/codemodder/providers/sarif/semgrep/SemgrepModule.java @@ -2,6 +2,7 @@ import com.contrastsecurity.sarif.Result; import com.contrastsecurity.sarif.SarifSchema210; +import com.google.inject.AbstractModule; import io.codemodder.*; import io.github.classgraph.*; import java.io.IOException; @@ -17,7 +18,7 @@ import org.slf4j.LoggerFactory; /** Responsible for binding Semgrep-related things. */ -public final class SemgrepModule extends CodemodCheckingAbstractModule { +public final class SemgrepModule extends AbstractModule { private final List> codemodTypes; private final Path codeDirectory; @@ -48,7 +49,6 @@ public SemgrepModule( final List> codemodTypes, final List sarifs, final SemgrepRuleFactory semgrepRuleFactory) { - super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.codeDirectory = Objects.requireNonNull(codeDirectory); this.includePatterns = Objects.requireNonNull(includePatterns); @@ -59,14 +59,7 @@ public SemgrepModule( } @Override - protected boolean isResponsibleFor(final Class codemod) { - Codemod annotation = codemod.getAnnotation(Codemod.class); - String id = annotation.id(); - return id.startsWith("semgrep:") || id.startsWith("pixee:") || id.startsWith("codemodder:"); - } - - @Override - protected void doConfigure() { + protected void configure() { // find all the @ProvidedSemgrepScan annotations and bind them as is Set packagesScanned = new HashSet<>(); @@ -148,6 +141,14 @@ protected void doConfigure() { } } + if (rules.isEmpty()) { + // no active codemods have rules to bind for, so we can return + LOG.info("No active codemods have Semgrep rules to bind for JIT scanning"); + return; + } else { + LOG.info("Found {} Semgrep rules to bind for JIT scanning", rules.size()); + } + /* * To avoid running semgrep and eating heavy, redundant file I/O for every codemod, we'll run it once with all rules, calculate which rules didn't "hit", and then storing an empty result for them. This will allow us to only run Semgrep on the rules for which we have evidence will hit. Given that we don't expect most projects to hit most codemods, this is a big time-savings. */ diff --git a/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java b/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java index fd8b91c33..44c7e6cd5 100644 --- a/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java +++ b/plugins/codemodder-plugin-sonar/src/main/java/io/codemodder/providers/sonar/SonarModule.java @@ -1,8 +1,7 @@ package io.codemodder.providers.sonar; +import com.google.inject.AbstractModule; import io.codemodder.CodeChanger; -import io.codemodder.Codemod; -import io.codemodder.CodemodCheckingAbstractModule; import io.codemodder.sonar.model.Hotspot; import io.codemodder.sonar.model.Issue; import io.codemodder.sonar.model.SonarFinding; @@ -15,7 +14,7 @@ import java.util.*; import javax.inject.Inject; -final class SonarModule extends CodemodCheckingAbstractModule { +final class SonarModule extends AbstractModule { private final List> codemodTypes; private final Path repository; @@ -28,7 +27,6 @@ final class SonarModule extends CodemodCheckingAbstractM final Path repository, final List findings, final Class> ruleFindingClass) { - super(codemodTypes); this.codemodTypes = Objects.requireNonNull(codemodTypes); this.repository = Objects.requireNonNull(repository); this.sonarFindings = findings; @@ -36,12 +34,7 @@ final class SonarModule extends CodemodCheckingAbstractM } @Override - protected boolean isResponsibleFor(final Class codemod) { - return codemod.getAnnotation(Codemod.class).id().startsWith("sonar:"); - } - - @Override - protected void doConfigure() { + protected void configure() { Map> findingsByRuleMap = groupFindingsByRule(sonarFindings);