diff --git a/MODULE.bazel b/MODULE.bazel index 86818c86da71..eb9440b19ce2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -73,7 +73,7 @@ maven.install( "org.jetbrains.kotlin:kotlin-stdlib:2.3.21", "org.jetbrains.kotlin:kotlin-compiler-embeddable:2.3.21", "org.jetbrains.kotlin:kotlin-allopen:2.3.21", - "org.jruby:jruby-complete:9.4.9.0", + "org.jruby:jruby-complete:10.0.6.0", "com.github.albfernandez:juniversalchardet:2.5.0", "io.joern:eclipse-cdt-core:9.2.100.202507101054+1", "org.eclipse.platform:org.eclipse.core.resources:3.23.0", @@ -128,7 +128,7 @@ JSSRC2CPG_ASTGEN_VERSION = "3.47.0" SWIFTSRC2CPG_ASTGEN_VERSION = "0.4.2" -RUBYSRC2CPG_ASTGEN_VERSION = "0.58.0" +RUBYSRC2CPG_ASTGEN_VERSION = "0.59.2" RUBYSRC2CPG_TYPE_STUBS_VERSION = "0.6.0" @@ -201,12 +201,47 @@ http_file( url = "https://github.com/joernio/astgen-monorepo/releases/download/swift-astgen%2Fv{}/SwiftAstGen-win.exe".format(SWIFTSRC2CPG_ASTGEN_VERSION), ) -#rubysrc2cpg astgen zip +#rubysrc2cpg astgen archive imports http_archive( - name = "rubysrc2cpg_astgen", + name = "rubysrc2cpg_astgen_macos_arm", build_file = "//joern-cli/frontends/rubysrc2cpg:rubysrc2cpg_astgen.BUILD", - sha256 = "66a2a61e822b1a4e391f86571b4a796c01c75b53c105fe544733b222843d06c7", - url = "https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen%2Fv{}/ruby_ast_gen_v{}.zip".format(RUBYSRC2CPG_ASTGEN_VERSION, RUBYSRC2CPG_ASTGEN_VERSION), + sha256 = "18c823845ef9ef5d064faad07f386e0849e09843544f0a433fd0228ae0392875", + url = "https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen%2Fv{}/ruby_ast_gen-macos-arm_v{}.zip".format(RUBYSRC2CPG_ASTGEN_VERSION, RUBYSRC2CPG_ASTGEN_VERSION), +) + +http_archive( + name = "rubysrc2cpg_astgen_macos_x86", + build_file = "//joern-cli/frontends/rubysrc2cpg:rubysrc2cpg_astgen.BUILD", + sha256 = "0d4c160e5f2f1f4d4de5749e6ffa21961cff4457d061a7176b8fd7682a37329c", + url = "https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen%2Fv{}/ruby_ast_gen-macos_v{}.zip".format(RUBYSRC2CPG_ASTGEN_VERSION, RUBYSRC2CPG_ASTGEN_VERSION), +) + +http_archive( + name = "rubysrc2cpg_astgen_linux_arm", + build_file = "//joern-cli/frontends/rubysrc2cpg:rubysrc2cpg_astgen.BUILD", + sha256 = "e6e15e10d8535cf20659502ac04589956dcd20f2ae027ff9cb20d731d8286e9a", + url = "https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen%2Fv{}/ruby_ast_gen-linux-arm_v{}.zip".format(RUBYSRC2CPG_ASTGEN_VERSION, RUBYSRC2CPG_ASTGEN_VERSION), +) + +http_archive( + name = "rubysrc2cpg_astgen_linux_x86", + build_file = "//joern-cli/frontends/rubysrc2cpg:rubysrc2cpg_astgen.BUILD", + sha256 = "6ced36c454474f72f25a0533dc6298f6ea4b7c6b05ea7964dad4796cafa033ba", + url = "https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen%2Fv{}/ruby_ast_gen-linux_v{}.zip".format(RUBYSRC2CPG_ASTGEN_VERSION, RUBYSRC2CPG_ASTGEN_VERSION), +) + +http_archive( + name = "rubysrc2cpg_astgen_win_arm", + build_file = "//joern-cli/frontends/rubysrc2cpg:rubysrc2cpg_astgen.BUILD", + sha256 = "d55f95e8174b204015639d545a9f16f80fe9c620e3291b99ad5c16d849d9001a", + url = "https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen%2Fv{}/ruby_ast_gen-win-arm_v{}.zip".format(RUBYSRC2CPG_ASTGEN_VERSION, RUBYSRC2CPG_ASTGEN_VERSION), +) + +http_archive( + name = "rubysrc2cpg_astgen_win_x86", + build_file = "//joern-cli/frontends/rubysrc2cpg:rubysrc2cpg_astgen.BUILD", + sha256 = "ecf7f126e47eeefb07bf3d72fa39ca3d3e403d50e91d26db0497c8930f7a10a4", + url = "https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen%2Fv{}/ruby_ast_gen-win_v{}.zip".format(RUBYSRC2CPG_ASTGEN_VERSION, RUBYSRC2CPG_ASTGEN_VERSION), ) http_file( diff --git a/joern-cli/frontends/rubysrc2cpg/BUILD b/joern-cli/frontends/rubysrc2cpg/BUILD index 3ac401a6b5c7..94866113dd4f 100644 --- a/joern-cli/frontends/rubysrc2cpg/BUILD +++ b/joern-cli/frontends/rubysrc2cpg/BUILD @@ -2,13 +2,33 @@ load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("//bazel:java.bzl", "java_binary") load("//bazel:scala.bzl", "scala_library", "scala_test") +filegroup( + name = "rubysrc2cpg_astgen", + srcs = select({ + "@bazel_tooling//platform:all": [ + "@rubysrc2cpg_astgen_linux_arm//:files", + "@rubysrc2cpg_astgen_linux_x86//:files", + "@rubysrc2cpg_astgen_macos_arm//:files", + "@rubysrc2cpg_astgen_macos_x86//:files", + "@rubysrc2cpg_astgen_win_arm//:files", + "@rubysrc2cpg_astgen_win_x86//:files", + ], + "@bazel_tooling//platform:macos-arm": ["@rubysrc2cpg_astgen_macos_arm//:files"], + "@bazel_tooling//platform:macos-x86": ["@rubysrc2cpg_astgen_macos_x86//:files"], + "@bazel_tooling//platform:linux-arm": ["@rubysrc2cpg_astgen_linux_arm//:files"], + "@bazel_tooling//platform:linux-x86": ["@rubysrc2cpg_astgen_linux_x86//:files"], + "@bazel_tooling//platform:win-arm": ["@rubysrc2cpg_astgen_win_x86//:files"], + "@bazel_tooling//platform:win-x86": ["@rubysrc2cpg_astgen_win_x86//:files"], + }), +) + scala_library( name = "rubysrc2cpg", srcs = glob([ "src/main/scala/**/*.scala", ]), data = [ - "@rubysrc2cpg_astgen//:files", + ":rubysrc2cpg_astgen", ], visibility = ["//visibility:public"], deps = [ diff --git a/joern-cli/frontends/rubysrc2cpg/build.sbt b/joern-cli/frontends/rubysrc2cpg/build.sbt index 083501a61a72..1e9d309cc820 100644 --- a/joern-cli/frontends/rubysrc2cpg/build.sbt +++ b/joern-cli/frontends/rubysrc2cpg/build.sbt @@ -40,6 +40,19 @@ astGenVersion := appProperties.value.getString("rubysrc2cpg.ruby_ast_gen_version lazy val astGenDlUrl = settingKey[String]("astgen download url") astGenDlUrl := s"https://github.com/joernio/astgen-monorepo/releases/download/ruby-astgen/v${astGenVersion.value}/" +lazy val astGenPlatformSuffix = settingKey[String]("platform suffix for ruby_ast_gen archive") +astGenPlatformSuffix := { + (Environment.operatingSystem, Environment.architecture) match { + case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.X86) => "macos" + case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.ARMv8) => "macos-arm" + case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.X86) => "linux" + case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.ARMv8) => "linux-arm" + case (Environment.OperatingSystemType.Windows, Environment.ArchitectureType.X86) => "win" + case (Environment.OperatingSystemType.Windows, Environment.ArchitectureType.ARMv8) => "win-arm" + case _ => "linux" + } +} + def hasCompatibleAstGenVersion(astGenBaseDir: File, astGenVersion: String): Boolean = { val versionFile = astGenBaseDir / "lib" / "ruby_ast_gen" / "version.rb" if (!versionFile.exists) return false @@ -56,9 +69,9 @@ def hasCompatibleAstGenVersion(astGenBaseDir: File, astGenVersion: String): Bool lazy val astGenResourceTask = taskKey[Seq[File]](s"Download `ruby_ast_gen` and package this under `resources`") astGenResourceTask := { val targetDir = baseDirectory.value / "src" / "main" / "resources" - val gemName = s"ruby_ast_gen_v${astGenVersion.value}.zip" + val gemName = s"ruby_ast_gen-${astGenPlatformSuffix.value}_v${astGenVersion.value}.zip" val compressGemPath = targetDir / gemName - val unpackedGemFullPath = targetDir / gemName.stripSuffix(s"_v${astGenVersion.value}.zip") + val unpackedGemFullPath = targetDir / "ruby_ast_gen" if (!hasCompatibleAstGenVersion(unpackedGemFullPath, astGenVersion.value)) { if (unpackedGemFullPath.exists()) IO.delete(unpackedGemFullPath) val url = s"${astGenDlUrl.value}$gemName" diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/resources/application.conf b/joern-cli/frontends/rubysrc2cpg/src/main/resources/application.conf index 55cdf1af0d4c..36960e6cd75a 100755 --- a/joern-cli/frontends/rubysrc2cpg/src/main/resources/application.conf +++ b/joern-cli/frontends/rubysrc2cpg/src/main/resources/application.conf @@ -1,4 +1,4 @@ rubysrc2cpg { - ruby_ast_gen_version: "0.58.0" + ruby_ast_gen_version: "0.59.2" joern_type_stubs_version: "0.6.0" } diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyAstGenRunner.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyAstGenRunner.scala index 4132bae45067..97baddbdd9df 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyAstGenRunner.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/parser/RubyAstGenRunner.scala @@ -5,7 +5,7 @@ import io.joern.rubysrc2cpg.parser.RubyAstGenRunner.{ExecutionEnvironment, JRuby import io.joern.x2cpg.SourceFiles import io.joern.x2cpg.astgen.AstGenRunner.{AstGenProgramMetaData, AstGenRunnerResult, DefaultAstGenRunnerResult} import io.joern.x2cpg.astgen.AstGenRunner -import io.joern.x2cpg.utils.JoernRunfilesLocator +import io.joern.x2cpg.utils.{Environment, JoernRunfilesLocator} import org.jruby.RubyInstanceConfig import org.jruby.embed.{LocalContextScope, LocalVariableBehavior, PathType, ScriptingContainer} import org.slf4j.LoggerFactory @@ -193,7 +193,11 @@ class RubyAstGenRunner(config: Config, sharedJRubyEnv: Option[JRubyEnvironment] object RubyAstGenRunner { private object astGenMetaData - extends AstGenProgramMetaData(name = "ruby_ast_gen", configPrefix = "rubysrc2cpg", multiArchitectureBuilds = true) + extends AstGenProgramMetaData( + name = "ruby_ast_gen", + configPrefix = "rubysrc2cpg", + multiArchitectureBuilds = false + ) /** Encapsulates the expensive JRuby runtime setup (execution environment and scripting container). Can be shared * across multiple RubyAstGenRunner instances to avoid repeated JRuby initialization. @@ -214,21 +218,39 @@ object RubyAstGenRunner { } object JRubyEnvironment { + private val logger = LoggerFactory.getLogger(getClass) + + private def platformSuffix: String = { + (Environment.operatingSystem, Environment.architecture) match { + case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.X86) => "linux_x86" + case (Environment.OperatingSystemType.Linux, Environment.ArchitectureType.ARMv8) => "linux_arm" + case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.X86) => "macos_x86" + case (Environment.OperatingSystemType.Mac, Environment.ArchitectureType.ARMv8) => "macos_arm" + case (Environment.OperatingSystemType.Windows, Environment.ArchitectureType.X86) => "win_x86" + case (Environment.OperatingSystemType.Windows, Environment.ArchitectureType.ARMv8) => "win_arm" + case _ => + logger.warn("Could not detect OS version! Defaulting to 'Linux'.") + "linux_x86" + } + } + def apply(): JRubyEnvironment = { val env = JoernRunfilesLocator - .resolve("rubysrc2cpg_astgen/") + .resolve(s"rubysrc2cpg_astgen_$platformSuffix/") .map(path => LocalDir(Path.of(path))) .getOrElse(prepareExecutionEnvironment("ruby_ast_gen")) - val cwd = env.path.toAbsolutePath.toString - val gemPath = Seq(cwd, "vendor", "bundle", "jruby", "3.1.0").mkString(separator) - val container = new ScriptingContainer(LocalContextScope.THREADSAFE, LocalVariableBehavior.TRANSIENT) - val config = container.getProvider.getRubyInstanceConfig + val cwd = env.path.toAbsolutePath.toString + val bundleBase = env.path.resolve("vendor").resolve("bundle").resolve("jruby") + val rubyAbi = Files.list(bundleBase).iterator.asScala.next().getFileName.toString + val gemPath = bundleBase.resolve(rubyAbi).toString + val container = new ScriptingContainer(LocalContextScope.THREADSAFE, LocalVariableBehavior.TRANSIENT) + val config = container.getProvider.getRubyInstanceConfig container.setCompileMode(RubyInstanceConfig.CompileMode.OFF) - container.setNativeEnabled(false) + container.setNativeEnabled(true) container.setObjectSpaceEnabled(true) container.setCurrentDirectory(cwd) - config.setLoadGemfile(true) - container.setEnvironment(Map("GEM_PATH" -> gemPath, "GEM_FILE" -> gemPath).asJava) + config.setLoadGemfile(false) + container.setEnvironment(Map("GEM_PATH" -> gemPath).asJava) config.setHasShebangLine(true) config.setHardExit(false) new JRubyEnvironment(env, container) diff --git a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/BooleanLogicTests.scala b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/BooleanLogicTests.scala index 52a086b83229..8dbe76223bcc 100644 --- a/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/BooleanLogicTests.scala +++ b/joern-cli/frontends/rubysrc2cpg/src/test/scala/io/joern/rubysrc2cpg/querying/BooleanLogicTests.scala @@ -136,6 +136,40 @@ class BooleanLogicTests extends RubyCode2CpgFixture { six.code shouldBe "6" } + "multi-line `&&` with operator at the start of a new line should be parsed as a single logical expression" in { + val cpg = code(""" + |1 + | && 2 + | && 3 + |""".stripMargin) + + val List(andOuter, andInner) = cpg.call.methodFullName(Operators.logicalAnd).l + val List(ab, c) = andOuter.argument.l + ab shouldBe andInner + c.code shouldBe "3" + + val List(a, b) = andInner.argument.l + a.code shouldBe "1" + b.code shouldBe "2" + } + + "multi-line `||` with operator at the start of a new line should be parsed as a single logical expression" in { + val cpg = code(""" + |1 + | || 2 + | || 3 + |""".stripMargin) + + val List(orOuter, orInner) = cpg.call.methodFullName(Operators.logicalOr).l + val List(ab, c) = orOuter.argument.l + ab shouldBe orInner + c.code shouldBe "3" + + val List(a, b) = orInner.argument.l + a.code shouldBe "1" + b.code shouldBe "2" + } + "`not` binds tighter than `or`" in { val cpg = code(""" |1 or not 2 diff --git a/maven_install.json b/maven_install.json index 746a8b76bd3a..f8e5bf5d6cfc 100644 --- a/maven_install.json +++ b/maven_install.json @@ -44,7 +44,7 @@ "org.jetbrains.kotlin:kotlin-stdlib": 181930543, "org.jetbrains.kotlin:kotlin-stdlib-jdk8": 1654605853, "org.jline:jline": 214767589, - "org.jruby:jruby-complete": -181731122, + "org.jruby:jruby-complete": -1254552563, "org.json4s:json4s-native_3": -1513317579, "org.ow2.asm:asm": 1206815935, "org.projectlombok:lombok": -1238622139, @@ -276,8 +276,7 @@ "org.jline:jline-terminal-jni:jar:sources": -1668611295, "org.jline:jline-terminal:jar:sources": -214275357, "org.jline:jline:jar:sources": 2039073485, - "org.jruby:jruby-complete": 1795657309, - "org.jruby:jruby-complete:jar:sources": 777557139, + "org.jruby:jruby-complete": -390576161, "org.json4s:json4s-ast_3": -507419474, "org.json4s:json4s-ast_3:jar:sources": 2122540071, "org.json4s:json4s-core_3": 1015651488, @@ -1108,10 +1107,9 @@ }, "org.jruby:jruby-complete": { "shasums": { - "jar": "c069cd9240c2a341e9df4aae55c54ac81577eb161d0afd6d026d6b3fafc7fefd", - "sources": "0056864423d4efb65013518e833704e4c9dcdd3f5235be7a8baa90811ea85c6b" + "jar": "b1a83ca46053894fcfd47df4094a21eac6db7fcba6fa6d99d77e391f620e85b7" }, - "version": "9.4.9.0" + "version": "10.0.6.0" }, "org.json4s:json4s-ast_3": { "shasums": { @@ -4256,12 +4254,11 @@ "org.jruby.ext.ripper", "org.jruby.ext.set", "org.jruby.ext.socket", - "org.jruby.ext.syslog", - "org.jruby.ext.tempfile", "org.jruby.ext.thread", "org.jruby.ext.timeout", "org.jruby.ext.tracepoint", "org.jruby.ext.zlib", + "org.jruby.gen", "org.jruby.internal.runtime", "org.jruby.internal.runtime.methods", "org.jruby.ir", @@ -4334,8 +4331,7 @@ "org.jruby.util.func", "org.jruby.util.io", "org.jruby.util.log", - "org.jruby.util.time", - "org.jruby.util.unsafe" + "org.jruby.util.time" ], "org.json4s:json4s-ast_3": [ "org.json4s", @@ -4730,7 +4726,6 @@ "org.jline:jline-terminal:jar:sources", "org.jline:jline:jar:sources", "org.jruby:jruby-complete", - "org.jruby:jruby-complete:jar:sources", "org.json4s:json4s-ast_3", "org.json4s:json4s-ast_3:jar:sources", "org.json4s:json4s-core_3", @@ -5001,7 +4996,6 @@ "org.jline:jline-terminal:jar:sources", "org.jline:jline:jar:sources", "org.jruby:jruby-complete", - "org.jruby:jruby-complete:jar:sources", "org.json4s:json4s-ast_3", "org.json4s:json4s-ast_3:jar:sources", "org.json4s:json4s-core_3", diff --git a/project/Versions.scala b/project/Versions.scala index b44ea9214f12..615417bb72f2 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -21,7 +21,7 @@ object Versions { val gradleTooling = "8.3" val jacksonDatabind = "2.17.0" val javaParser = "3.28.0" - val jRuby = "9.4.9.0" + val jRuby = "10.0.6.0" val json4s = "4.0.7" val lombok = "1.18.42" val mavenArtifact = "3.9.16"