From 6c76a1a683620fe25fbdb074b861cc6e3b5f177c Mon Sep 17 00:00:00 2001 From: andrecs <12188364+andrecsilva@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:56:13 -0300 Subject: [PATCH 1/5] Forced use of Woodstox parser for FormatCommand and fixed bug --- gradle/libs.versions.toml | 2 ++ .../codemodder-plugin-maven/build.gradle.kts | 1 + .../plugins/maven/MavenProvider.java | 2 +- .../plugins/maven/operator/FormatCommand.java | 19 ++++++++++++++++--- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 151a1040b..a4ca3f94c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,6 +15,7 @@ picocli = "4.7.0" slf4j = "2.0.6" guice = "5.1.0" dom4j = "2.1.4" +woodstox = "7.1.0" [libraries] autovalue-annotations = { module = "com.google.auto.value:auto-value-annotations", version.ref = "auto-value" } @@ -27,6 +28,7 @@ contrast-sarif = "com.contrastsecurity:java-sarif:2.0" gson = "com.google.code.gson:gson:2.9.0" guice = { module = "com.google.inject:guice", version.ref = "guice" } immutables = "org.immutables:value:2.9.0" +woodstox = { module = "com.fasterxml.woodstox:woodstox-core", version.ref = "woodstox" } jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "jackson" } jackson-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", version.ref = "jackson" } javadiff = "io.github.java-diff-utils:java-diff-utils:4.12" diff --git a/plugins/codemodder-plugin-maven/build.gradle.kts b/plugins/codemodder-plugin-maven/build.gradle.kts index 754a60150..a385b4d50 100644 --- a/plugins/codemodder-plugin-maven/build.gradle.kts +++ b/plugins/codemodder-plugin-maven/build.gradle.kts @@ -31,4 +31,5 @@ dependencies { implementation(libs.diff.match.patch) implementation(libs.slf4j.simple) implementation(libs.slf4j.api) + implementation(libs.woodstox) } diff --git a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/MavenProvider.java b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/MavenProvider.java index 72dce755f..ae470e984 100644 --- a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/MavenProvider.java +++ b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/MavenProvider.java @@ -18,7 +18,7 @@ * *

a. We skip parent finding if there's not a relativePath declaration (this is by design), so * sometimes pom finding will fail on purpose b. there are several flags on ProjectModelFactory - * which aren't applied. They relate to verisons, upgrading and particularly: Actives Profiles c. If + * which aren't applied. They relate to versions, upgrading and particularly: Actives Profiles c. If * you need anything declared in a ~/.m2/settings.xml, we don't support that (e.g., passwords or * proxies) d. Haven't tested, but I'm almost sure that it wouldn't work on any repo other than * central e. We allow on this module to do online resolution. HOWEVER by default its offline f. You diff --git a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java index 4ada1c5a7..2868e12d8 100644 --- a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java +++ b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java @@ -2,6 +2,8 @@ import static io.github.pixee.security.XMLInputFactorySecurity.hardenFactory; +import com.ctc.wstx.evt.CompactStartElement; +import com.ctc.wstx.stax.WstxInputFactory; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -38,7 +40,7 @@ class FormatCommand extends AbstractCommand { private static final Logger LOGGER = LoggerFactory.getLogger(FormatCommand.class); /** StAX InputFactory */ - private XMLInputFactory inputFactory = hardenFactory(XMLInputFactory.newInstance()); + private XMLInputFactory inputFactory = WstxInputFactory.newInstance(); /** StAX OutputFactory */ private XMLOutputFactory outputFactory = XMLOutputFactory.newInstance(); @@ -270,6 +272,10 @@ private void parseXmlAndCharset(POMDocument pomFile) throws XMLStreamException, int elementStart = 0; List prevEvents = new ArrayList<>(); + System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); + System.out.println(inputFactory.getClass()); + System.out.println(eventReader.getClass()); + System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); while (eventReader.hasNext()) { XMLEvent event = eventReader.nextEvent(); @@ -324,8 +330,15 @@ private void parseXmlAndCharset(POMDocument pomFile) throws XMLStreamException, String originalPomCharsetString = new String(pomFile.getOriginalPom(), pomFile.getCharset()); - String untrimmedOriginalContent = - originalPomCharsetString.substring(elementStart, offset); + String untrimmedOriginalContent = ""; + // is self closing element, tag is contained within the offset of the next element + if (prevEvents.get(prevEvents.size() - 1) instanceof CompactStartElement) { + untrimmedOriginalContent = + originalPomCharsetString.substring( + offset, eventReader.peek().getLocation().getCharacterOffset()); + } else { + untrimmedOriginalContent = originalPomCharsetString.substring(elementStart, offset); + } String trimmedOriginalContent = untrimmedOriginalContent.trim(); From 44dc4f9f7310aa650a62702fe998273b2310e235 Mon Sep 17 00:00:00 2001 From: andrecs <12188364+andrecsilva@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:59:39 -0300 Subject: [PATCH 2/5] fixup! Forced use of Woodstox parser for FormatCommand and fixed bug --- .../codemodder/plugins/maven/operator/FormatCommand.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java index 2868e12d8..ef792d679 100644 --- a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java +++ b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java @@ -4,6 +4,7 @@ import com.ctc.wstx.evt.CompactStartElement; import com.ctc.wstx.stax.WstxInputFactory; +import com.ctc.wstx.stax.WstxOutputFactory; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -43,7 +44,7 @@ class FormatCommand extends AbstractCommand { private XMLInputFactory inputFactory = WstxInputFactory.newInstance(); /** StAX OutputFactory */ - private XMLOutputFactory outputFactory = XMLOutputFactory.newInstance(); + private XMLOutputFactory outputFactory = WstxOutputFactory.newInstance(); private List singleElementsWithAttributes = new ArrayList<>(); @@ -272,10 +273,6 @@ private void parseXmlAndCharset(POMDocument pomFile) throws XMLStreamException, int elementStart = 0; List prevEvents = new ArrayList<>(); - System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); - System.out.println(inputFactory.getClass()); - System.out.println(eventReader.getClass()); - System.out.println("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="); while (eventReader.hasNext()) { XMLEvent event = eventReader.nextEvent(); @@ -331,7 +328,7 @@ private void parseXmlAndCharset(POMDocument pomFile) throws XMLStreamException, new String(pomFile.getOriginalPom(), pomFile.getCharset()); String untrimmedOriginalContent = ""; - // is self closing element, tag is contained within the offset of the next element + // is self-closing element, tag is contained within the offset of the next element if (prevEvents.get(prevEvents.size() - 1) instanceof CompactStartElement) { untrimmedOriginalContent = originalPomCharsetString.substring( From 178209642fc09cb22bd12a1a9a98746151aea055 Mon Sep 17 00:00:00 2001 From: andrecs <12188364+andrecsilva@users.noreply.github.com> Date: Thu, 5 Dec 2024 10:25:08 -0300 Subject: [PATCH 3/5] fixup! fixup! Forced use of Woodstox parser for FormatCommand and fixed bug --- .../plugins/maven/operator/FormatCommand.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java index ef792d679..143b28a2b 100644 --- a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java +++ b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java @@ -2,7 +2,6 @@ import static io.github.pixee.security.XMLInputFactorySecurity.hardenFactory; -import com.ctc.wstx.evt.CompactStartElement; import com.ctc.wstx.stax.WstxInputFactory; import com.ctc.wstx.stax.WstxOutputFactory; import java.io.ByteArrayInputStream; @@ -327,14 +326,26 @@ private void parseXmlAndCharset(POMDocument pomFile) throws XMLStreamException, String originalPomCharsetString = new String(pomFile.getOriginalPom(), pomFile.getCharset()); + var prev = prevEvents.get(prevEvents.size() - 1); String untrimmedOriginalContent = ""; // is self-closing element, tag is contained within the offset of the next element - if (prevEvents.get(prevEvents.size() - 1) instanceof CompactStartElement) { + if (prev instanceof StartElement + && prev.getLocation().getCharacterOffset() + == endElementEvent.getLocation().getCharacterOffset()) { untrimmedOriginalContent = originalPomCharsetString.substring( offset, eventReader.peek().getLocation().getCharacterOffset()); } else { - untrimmedOriginalContent = originalPomCharsetString.substring(elementStart, offset); + // is empty tag, the last character events is not in between the tags + if (prev.isStartElement()) { + untrimmedOriginalContent = + originalPomCharsetString.substring( + prev.getLocation().getCharacterOffset(), + eventReader.peek().getLocation().getCharacterOffset()); + + } else { + untrimmedOriginalContent = originalPomCharsetString.substring(elementStart, offset); + } } String trimmedOriginalContent = untrimmedOriginalContent.trim(); From 000b5e5e7611b9f5366b68e99455522ec4dd5bfe Mon Sep 17 00:00:00 2001 From: andrecs <12188364+andrecsilva@users.noreply.github.com> Date: Fri, 6 Dec 2024 06:49:14 -0300 Subject: [PATCH 4/5] Fixed issue with some codemods that target XML files --- .../io/codemodder/DefaultXPathStreamProcessor.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/framework/codemodder-base/src/main/java/io/codemodder/DefaultXPathStreamProcessor.java b/framework/codemodder-base/src/main/java/io/codemodder/DefaultXPathStreamProcessor.java index 6fc938056..4898e1e43 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/DefaultXPathStreamProcessor.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/DefaultXPathStreamProcessor.java @@ -73,9 +73,14 @@ public Optional process( XMLEventWriter xmlWriter = outputFactory.createXMLEventWriter(sw); while (xmlReader.hasNext()) { final XMLEvent currentEvent = xmlReader.nextEvent(); - Location location = currentEvent.getLocation(); - if (doesPositionMatch(httpMethodPositions, location)) { - handler.handle(xmlReader, xmlWriter, currentEvent); + // get the position of the last character of the event, that is, the start of the next one + if (xmlReader.hasNext()) { + Location location = xmlReader.peek().getLocation(); + if (doesPositionMatch(httpMethodPositions, location)) { + handler.handle(xmlReader, xmlWriter, currentEvent); + } else { + xmlWriter.add(currentEvent); + } } else { xmlWriter.add(currentEvent); } From 721ef60b1dda38db93d819739b21d8e074f3f342 Mon Sep 17 00:00:00 2001 From: andrecs <12188364+andrecsilva@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:37:51 -0300 Subject: [PATCH 5/5] Changed explicit instantiation --- .../io/codemodder/plugins/maven/operator/FormatCommand.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java index 143b28a2b..96b5810c7 100644 --- a/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java +++ b/plugins/codemodder-plugin-maven/src/main/java/io/codemodder/plugins/maven/operator/FormatCommand.java @@ -2,8 +2,6 @@ import static io.github.pixee.security.XMLInputFactorySecurity.hardenFactory; -import com.ctc.wstx.stax.WstxInputFactory; -import com.ctc.wstx.stax.WstxOutputFactory; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -40,10 +38,10 @@ class FormatCommand extends AbstractCommand { private static final Logger LOGGER = LoggerFactory.getLogger(FormatCommand.class); /** StAX InputFactory */ - private XMLInputFactory inputFactory = WstxInputFactory.newInstance(); + private XMLInputFactory inputFactory = XMLInputFactory.newInstance().newInstance(); /** StAX OutputFactory */ - private XMLOutputFactory outputFactory = WstxOutputFactory.newInstance(); + private XMLOutputFactory outputFactory = XMLOutputFactory.newInstance(); private List singleElementsWithAttributes = new ArrayList<>();