From 9352de52704169f41fb983843e6b80e5a4fdeb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E6=8C=AF=E5=AE=87?= <> Date: Fri, 7 Feb 2025 15:57:10 +0800 Subject: [PATCH] fix(pipeline): update SourceFetcher to use dynamic git credentials ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 孙振宇 <> --- .../vars/executeFreeleapsPipeline.groovy | 388 +++++++++--------- 1 file changed, 200 insertions(+), 188 deletions(-) diff --git a/first-class-pipeline/vars/executeFreeleapsPipeline.groovy b/first-class-pipeline/vars/executeFreeleapsPipeline.groovy index f1514a9d..068cb73d 100644 --- a/first-class-pipeline/vars/executeFreeleapsPipeline.groovy +++ b/first-class-pipeline/vars/executeFreeleapsPipeline.groovy @@ -15,32 +15,34 @@ import com.freeleaps.devops.enums.CodeLinterTypes import com.freeleaps.devops.enums.ImageBuilderTypes def generateComponentStages(component, configurations) { - return [ - stage("${component.name} :: Build Agent Setup") { - script { - if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { - def buildAgentImage = component.buildAgentImage - if (buildAgentImage == null || buildAgentImage.isEmpty()) { - log.warn("Pipeline", "Not set buildAgentImage for ${component.name}, using default build agent image") + def stages = [] + stages.addAll([ + // Build Agent Setup + stage("${component.name} :: Build Agent Setup") { + script { + if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { + def buildAgentImage = component.buildAgentImage + if (buildAgentImage == null || buildAgentImage.isEmpty()) { + log.warn("Pipeline", "Not set buildAgentImage for ${component.name}, using default build agent image") - def language = ServiceLanguage.parse(component.language) - switch(language) { - case ServiceLanguage.PYTHON: - buildAgentImage = "docker.io/python:3.10-slim-buster" - break - case ServiceLanguage.JS: - buildAgentImage = "docker.io/node:lts-alpine" - break - default: - error("Unknown service language") - } - } - log.info("Pipeline", "Using ${buildAgentImage} as build agent image for ${component.name}") - env.buildAgentImage = buildAgentImage - } - } + def language = ServiceLanguage.parse(component.language) + switch(language) { + case ServiceLanguage.PYTHON: + buildAgentImage = "docker.io/python:3.10-slim-buster" + break + case ServiceLanguage.JS: + buildAgentImage = "docker.io/node:lts-alpine" + break + default: + error("Unknown service language") + } + } + log.info("Pipeline", "Using ${buildAgentImage} as build agent image for ${component.name}") + env.buildAgentImage = buildAgentImage + } + } }, - + // Dependencies Resolving stage("${component.name} :: Dependencies Resolving") { podTemplate( label: "dep-resolver-${component.name}", @@ -81,195 +83,208 @@ def generateComponentStages(component, configurations) { } } }, + ]) - stage("${component.name} :: Code Linter Preparation") { - script { - if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { - if (component.lintEnabled != null && component.lintEnabled) { - log.info("Pipeline", "Code linting has enabled, preparing linter...") + if (component.lintEnabled != null && component.lintEnabled) { + stages.addAll([ + // Code Linter Environment Preparation + stage("${component.name} :: Code Linter Preparation") { + script { + if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { + if (component.lintEnabled != null && component.lintEnabled) { + log.info("Pipeline", "Code linting has enabled, preparing linter...") - if (component.linter == null || component.linter.isEmpty()) { - log.error("Pipeline", "Not set linter for ${component.name}, using default linter settings as fallback") - } - - def linter = CodeLinterTypes.parse(component.linter) - - if (linter == null) { - log.error("Pipeline", "Unknown linter for ${component.name}, skipping code linting") - } - - if (linter.language != ServiceLanguage.parse(component.language)) { - log.error("Pipeline", "Linter ${linter.linter} is not supported for ${component.language}, skipping code linting") - } - - log.info("Pipeline", "Using ${linter.linter} with image ${linter.containerImage} as linter for ${component.name}") - env.linterContainerImage = linter.containerImage - } else { - log.info("Pipeline", "Code linting is not enabled for ${component.name}, skipping...") - } - } - } - }, - - stage("${component.name} :: Code Linting If Enabled") { - when { - expression { - return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.linterContainerImage != null && !env.linterContainerImage.isEmpty() - } - } - podTemplate( - label: "code-linter-${component.name}", - containers: [ - containerTemplate( - name: 'code-linter', - image: env.linterContainerImage, - ttyEnabled: true, - command: 'sleep', - args: 'infinity' - ) - ] - ) { - node("code-linter-${component.name}") { - container('code-linter') { - script { - if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { - if (component.lintEnabled != null && component.lintEnabled) { - log.info("Pipeline", "Code linting has enabled, linting code...") - - def sourceFetcher = new SourceFetcher(this) - sourceFetcher.fetch(configurations) - - def linterType = CodeLinterTypes.parse(component.linter) - - def codeLintExecutor = new CodeLintExecutor(this, env.workroot + "/" + component.root + "/", component.linterConfig, linterType) - codeLintExecutor.execute() - } + if (component.linter == null || component.linter.isEmpty()) { + log.error("Pipeline", "Not set linter for ${component.name}, using default linter settings as fallback") } - } - } - } - } - }, - stage("${component.name} :: SAST Scanner Preparation") { - script { - if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { - if (component.sastEnabled != null && component.sastEnabled) { - log.info("Pipeline", "SAST scanning has enabled, preparing scanner...") + def linter = CodeLinterTypes.parse(component.linter) - if (sastScanner == null || sastScanner.isEmpty()) { - log.error("Pipeline", "Not set sastScanner for ${component.name}") - } + if (linter == null) { + log.error("Pipeline", "Unknown linter for ${component.name}, skipping code linting") + } - def sastScannerType = SASTScannerTypes.parse(component.sastScanner) - if (sastScannerType == null) { - log.error("Pipeline", "Unknown SAST scanner for ${component.name}, skipping SAST scanning") - } else if (sastScannerType.language != ServiceLanguage.parse(component.language)) { - log.error("Pipeline", "SAST scanner ${sastScannerType.scanner} is not supported for ${component.language}, skipping SAST scanning") + if (linter.language != ServiceLanguage.parse(component.language)) { + log.error("Pipeline", "Linter ${linter.linter} is not supported for ${component.language}, skipping code linting") + } + + log.info("Pipeline", "Using ${linter.linter} with image ${linter.containerImage} as linter for ${component.name}") + env.linterContainerImage = linter.containerImage } else { - log.info("Pipeline", "Using ${sastScanner} as SAST scanner for ${component.name}") - env.sastScannerContainerImage = sastScannerType.containerImage + log.info("Pipeline", "Code linting is not enabled for ${component.name}, skipping...") } } } - } - }, + }, + // Code Linting + stage("${component.name} :: Code Linting") { + podTemplate( + label: "code-linter-${component.name}", + containers: [ + containerTemplate( + name: 'code-linter', + image: env.linterContainerImage, + ttyEnabled: true, + command: 'sleep', + args: 'infinity' + ) + ] + ) { + node("code-linter-${component.name}") { + container('code-linter') { + script { + if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { + if (component.lintEnabled != null && component.lintEnabled) { + log.info("Pipeline", "Code linting has enabled, linting code...") - stage("${component.name} :: SAST Scanning If Enabled") { - when { - expression { - return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.sastScannerContainerImage != null && !env.sastScannerContainerImage.isEmpty() - } - } - podTemplate( - label: "sast-scanner-${component.name}", - containers: [ - containerTemplate( - name: 'sast-scanner', - image: env.sastScannerContainerImage, - ttyEnabled: true, - command: 'sleep', - args: 'infinity' - ) - ] - ) { - node("sast-scanner-${component.name}") { - container('sast-scanner') { - script { - if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { - if (component.sastEnabled != null && component.sastEnabled) { - log.info("Pipeline", "SAST scanning has enabled, scanning code...") + def sourceFetcher = new SourceFetcher(this) + sourceFetcher.fetch(configurations) - def sourceFetcher = new SourceFetcher(this) - sourceFetcher.fetch(configurations) + def linterType = CodeLinterTypes.parse(component.linter) - def sastScannerType = SASTScannerTypes.parse(component.sastScanner) - - def sastScanner = new SASTExecutor(this, env.workroot + "/" + component.root + "/", sastScannerType) - sastScanner.scan() + def codeLintExecutor = new CodeLintExecutor(this, env.workroot + "/" + component.root + "/", component.linterConfig, linterType) + codeLintExecutor.execute() + } } } } } } } - }, + ]) + } - stage("${component.name} :: Semantic Release Preparation") { - script { - if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { - if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { - log.info("Pipeline", "Semantic releasing has enabled, preparing semantic release...") + if (component.sastEnabled != null && component.sastEnabled) { + stages.addAll([ + // SAST Scanner Environment Preparation + stage("${component.name} :: SAST Scanner Preparation") { + script { + if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { + if (component.sastEnabled != null && component.sastEnabled) { + log.info("Pipeline", "SAST scanning has enabled, preparing scanner...") - if (component.semanticReleaseBranch == null || component.semanticReleaseBranch.isEmpty()) { - log.error("Pipeline", "Not set semanticReleaseBranch for ${component.name}, please set it to enable semantic release") + if (sastScanner == null || sastScanner.isEmpty()) { + log.error("Pipeline", "Not set sastScanner for ${component.name}") + } + + def sastScannerType = SASTScannerTypes.parse(component.sastScanner) + if (sastScannerType == null) { + log.error("Pipeline", "Unknown SAST scanner for ${component.name}, skipping SAST scanning") + } else if (sastScannerType.language != ServiceLanguage.parse(component.language)) { + log.error("Pipeline", "SAST scanner ${sastScannerType.scanner} is not supported for ${component.language}, skipping SAST scanning") + } else { + log.info("Pipeline", "Using ${sastScanner} as SAST scanner for ${component.name}") + env.sastScannerContainerImage = sastScannerType.containerImage + } } - - log.info("Pipeline", "Using ${component.semanticReleaseBranch} as semantic release branch for ${component.name}") - env.semanticReleasingContainerImage = "docker.io/semantic-release/semantic-release:latest" } } - } - }, - - stage("${component.name} :: Semantic Releasing If Enabled") { - when { - expression { - return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.semanticReleasingContainerImage != null && !env.semanticReleasingContainerImage.isEmpty() + }, + // SAST Scanning + stage("${component.name} :: SAST Scanning") { + when { + expression { + return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.sastScannerContainerImage != null && !env.sastScannerContainerImage.isEmpty() + } } - } - podTemplate( - label: "semantic-releasing-${component.name}", - containers: [ - containerTemplate( - name: 'semantic-releasing', - image: env.semanticReleasingContainerImage, - ttyEnabled: true, - command: 'sleep', - args: 'infinity' - ) - ] - ) { - node("semantic-releasing-${component.name}") { - container('semantic-releasing') { - script { - if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { - if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { - log.info("Pipeline", "Semantic releasing has enabled, releasing...") + podTemplate( + label: "sast-scanner-${component.name}", + containers: [ + containerTemplate( + name: 'sast-scanner', + image: env.sastScannerContainerImage, + ttyEnabled: true, + command: 'sleep', + args: 'infinity' + ) + ] + ) { + node("sast-scanner-${component.name}") { + container('sast-scanner') { + script { + if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { + if (component.sastEnabled != null && component.sastEnabled) { + log.info("Pipeline", "SAST scanning has enabled, scanning code...") - def sourceFetcher = new SourceFetcher(this) - sourceFetcher.fetch(configurations) + def sourceFetcher = new SourceFetcher(this) + sourceFetcher.fetch(configurations) - def semanticReleasingExecutor = new SemanticReleasingExecutor(this, env.workroot + "/" + component.root + "/") - semanticReleasingExecutor.release() + def sastScannerType = SASTScannerTypes.parse(component.sastScanner) + + def sastScanner = new SASTExecutor(this, env.workroot + "/" + component.root + "/", sastScannerType) + sastScanner.scan() + } } } } } } } - }, + ]) + } + if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { + stages.addAll([ + // Semantic Release Environment Preparation + stage("${component.name} :: Semantic Release Preparation") { + script { + if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { + if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { + log.info("Pipeline", "Semantic releasing has enabled, preparing semantic release...") + + if (component.semanticReleaseBranch == null || component.semanticReleaseBranch.isEmpty()) { + log.error("Pipeline", "Not set semanticReleaseBranch for ${component.name}, please set it to enable semantic release") + } + + log.info("Pipeline", "Using ${component.semanticReleaseBranch} as semantic release branch for ${component.name}") + env.semanticReleasingContainerImage = "docker.io/semantic-release/semantic-release:latest" + } + } + } + }, + // Semantic Releasing + stage("${component.name} :: Semantic Releasing") { + when { + expression { + return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.semanticReleasingContainerImage != null && !env.semanticReleasingContainerImage.isEmpty() + } + } + podTemplate( + label: "semantic-releasing-${component.name}", + containers: [ + containerTemplate( + name: 'semantic-releasing', + image: env.semanticReleasingContainerImage, + ttyEnabled: true, + command: 'sleep', + args: 'infinity' + ) + ] + ) { + node("semantic-releasing-${component.name}") { + container('semantic-releasing') { + script { + if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { + if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { + log.info("Pipeline", "Semantic releasing has enabled, releasing...") + + def sourceFetcher = new SourceFetcher(this) + sourceFetcher.fetch(configurations) + + def semanticReleasingExecutor = new SemanticReleasingExecutor(this, env.workroot + "/" + component.root + "/") + semanticReleasingExecutor.release() + } + } + } + } + } + } + } + ]) + } + + stages.addAll([ + // Compilation & Packaging stage("${component.name} :: Compilation & Packaging") { podTemplate( label: "build-agent-${component.name}", @@ -313,13 +328,13 @@ def generateComponentStages(component, configurations) { stash includes: artifact, name: "${component.name}-${artifact}" } } + } } } } } - } - }, - + }, + // Image Builder Setup stage("${component.name} :: Image Builder Setup") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { @@ -340,7 +355,7 @@ def generateComponentStages(component, configurations) { } } }, - + // Image Building & Publishing stage("${component.name} :: Image Building & Publishing") { when { expression { @@ -400,11 +415,8 @@ def generateComponentStages(component, configurations) { } } } - - // stage("${component.name} :: Argo Application Version Updating (Deploying)") { - - // } - ] + ]) + return stages } def call(Closure closure) {