Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[c#] refactor local summary creator into its own util #5236

Merged
merged 1 commit into from
Jan 19, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ import io.joern.csharpsrc2cpg.astcreation.AstCreator
import io.joern.csharpsrc2cpg.datastructures.CSharpProgramSummary
import io.joern.csharpsrc2cpg.parser.DotNetJsonParser
import io.joern.csharpsrc2cpg.passes.{AstCreationPass, DependencyPass}
import io.joern.csharpsrc2cpg.utils.{DependencyDownloader, DotNetAstGenRunner}
import io.joern.csharpsrc2cpg.utils.{DependencyDownloader, DotNetAstGenRunner, ProgramSummaryCreator}
import io.joern.x2cpg.X2Cpg.withNewEmptyCpg
import io.joern.x2cpg.astgen.AstGenRunner.AstGenRunnerResult
import io.joern.x2cpg.astgen.ParserResult
@@ -35,13 +35,7 @@ class CSharpSrc2Cpg extends X2CpgFrontend[Config] {
File.usingTemporaryDirectory("csharpsrc2cpgOut") { tmpDir =>
val astGenResult = new DotNetAstGenRunner(config).execute(tmpDir)
val astCreators = CSharpSrc2Cpg.processAstGenRunnerResults(astGenResult.parsedFiles, config)
// Pre-parse the AST creators for high level structures
val internalProgramSummary = summarizeAstCreators(astCreators)
// Load built-in and/or external summaries for missing imports
val builtinSummary = createBuiltinSummary(config, internalProgramSummary)
val internalAndBuiltinSummary = internalProgramSummary ++= builtinSummary
val externalSummary = createExternalSummary(config, internalAndBuiltinSummary)
val localSummary = internalAndBuiltinSummary ++= externalSummary
val localSummary = ProgramSummaryCreator.from(astCreators, config)

val hash = HashUtil.sha256(astCreators.map(_.parserResult).map(x => Paths.get(x.fullPath)))
new MetaDataPass(cpg, Languages.CSHARPSRC, config.inputPath, Option(hash)).createAndApply()
@@ -61,33 +55,6 @@ class CSharpSrc2Cpg extends X2CpgFrontend[Config] {
}
}

private def summarizeAstCreators(astCreators: Seq[AstCreator]): CSharpProgramSummary = {
ConcurrentTaskUtil
.runUsingThreadPool(astCreators.map(x => () => x.summarize()).iterator)
.flatMap {
case Failure(exception) =>
logger.warn(s"Unable to pre-parse C# file, skipping - ", exception)
None
case Success(summary) => Option(summary)
}
.foldLeft(CSharpProgramSummary(imports = CSharpProgramSummary.initialImports))(_ ++= _)
}

private def createBuiltinSummary(config: Config, internalSummary: CSharpProgramSummary): CSharpProgramSummary = {
Option
.when(config.useBuiltinSummaries)(CSharpProgramSummary.builtinTypesSummary)
.map(_.filter(namespacePred = (ns, _) => internalSummary.imports.contains(ns)))
.getOrElse(CSharpProgramSummary())
}

private def createExternalSummary(config: Config, internalSummary: CSharpProgramSummary): CSharpProgramSummary = {
val jsonFilePaths = config.externalSummaryPaths
Option
.when(jsonFilePaths.nonEmpty)(CSharpProgramSummary.externalTypesSummary(jsonFilePaths))
.map(_.filter(namespacePred = (ns, _) => internalSummary.imports.contains(ns)))
.getOrElse(CSharpProgramSummary())
}

private def buildFiles(config: Config): List[String] = {
SourceFiles.determine(
config.inputPath,
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package io.joern.csharpsrc2cpg.utils

import io.joern.csharpsrc2cpg.Config
import io.joern.csharpsrc2cpg.astcreation.AstCreator
import io.joern.csharpsrc2cpg.datastructures.CSharpProgramSummary
import io.joern.x2cpg.utils.ConcurrentTaskUtil
import org.slf4j.LoggerFactory

import scala.util.{Failure, Success}

/** Builds a `CSharpProgramSummary` by pre-parsing AST creators for high level structures, taking into account related
* frontend options.
*/
object ProgramSummaryCreator {

private val logger = LoggerFactory.getLogger(getClass)

def from(astCreators: Seq[AstCreator], config: Config): CSharpProgramSummary = {
val internalSummary = summarizeAstCreators(astCreators)
val externalSummary = buildExternalSummary(config.useBuiltinSummaries, config.externalSummaryPaths)
internalSummary ++= externalSummary.filter(namespacePred = (ns, _) => internalSummary.imports.contains(ns))
}

private def summarizeAstCreators(astCreators: Seq[AstCreator]): CSharpProgramSummary = {
ConcurrentTaskUtil
.runUsingThreadPool(astCreators.map(x => () => x.summarize()).iterator)
.flatMap {
case Failure(exception) =>
logger.warn(s"Unable to pre-parse C# file, skipping - ", exception)
None
case Success(summary) => Option(summary)
}
.foldLeft(CSharpProgramSummary(imports = CSharpProgramSummary.initialImports))(_ ++= _)
}

private def buildExternalSummary(withBuiltinTypes: Boolean, withJsonFiles: Set[String]): CSharpProgramSummary = {
val builtin = if withBuiltinTypes then CSharpProgramSummary.builtinTypesSummary else CSharpProgramSummary()
val fromJson =
if withJsonFiles.nonEmpty then CSharpProgramSummary.externalTypesSummary(withJsonFiles)
else CSharpProgramSummary()
builtin ++= fromJson
}
}