1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-14 11:03:09 +02:00
sap-jenkins-library/cmd/piper.go

595 lines
24 KiB
Go
Raw Normal View History

package cmd
import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/orchestrator"
"github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/pkg/errors"
"github.com/spf13/cobra"
)
// GeneralConfigOptions contains all global configuration options for piper binary
type GeneralConfigOptions struct {
GitHubAccessTokens map[string]string // map of tokens with url as key in order to maintain url-specific tokens
CorrelationID string
CustomConfig string
GitHubTokens []string // list of entries in form of <server>:<token> to allow token authentication for downloading config / defaults
DefaultConfig []string //ordered list of Piper default configurations. Can be filePath or ENV containing JSON in format 'ENV:MY_ENV_VAR'
IgnoreCustomDefaults bool
ParametersJSON string
EnvRootPath string
NoTelemetry bool
StageName string
StepConfigJSON string
StepMetadata string //metadata to be considered, can be filePath or ENV containing JSON in format 'ENV:MY_ENV_VAR'
StepName string
Verbose bool
LogFormat string
VaultRoleID string
VaultRoleSecretID string
VaultToken string
VaultServerURL string
VaultNamespace string
VaultPath string
HookConfig HookConfiguration
MetaDataResolver func() map[string]config.StepData
feat(gcs): allow upload to gcs from steps (#3034) * Upload reports to Google Cloud Storage bucket * Added tests. Made fixes * Update step generation. GCS client was moved to GeneralConfig * Code was refactored * Fixed issues * Fixed issues * Code correction due to PR comments * Improved gcs client and integration tests * Integrated gcp config. Updated step metadata * Fixed issues. Added tests * Added cpe, vault, aliases resolving for reporting parameters * Added tests * Uncommented DeferExitHandler. Removed useless comments * fixed cloning of config * Added comments for exported functions. Removed unused mock * minor fix * Implemented setting of report name via paramRef * some refactoring. Writing tests * Update pkg/config/reporting.go * Update cmd/sonarExecuteScan_generated.go * Apply suggestions from code review * Update pkg/config/reporting.go * Update pkg/config/reporting.go * fixed removing valut secret files * Update pkg/config/reporting.go * restore order * restore order * Apply suggestions from code review * go generate * fixed tests * Update resources/metadata/sonarExecuteScan.yaml * Update resources.go * Fixed tests. Code was regenerated * changed somewhere gcp to gcs. Fixed one test * move gcsSubFolder to input parameters * fixed removing valut secret files * minor fix in integration tests * fix integration tests Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
2021-12-15 16:07:47 +02:00
GCPJsonKeyFilePath string
GCSFolderPath string
GCSBucketId string
GCSSubFolder string
}
// HookConfiguration contains the configuration for supported hooks, so far Sentry and Splunk are supported.
type HookConfiguration struct {
SentryConfig SentryConfiguration `json:"sentry,omitempty"`
SplunkConfig SplunkConfiguration `json:"splunk,omitempty"`
}
// SentryConfiguration defines the configuration options for the Sentry logging system
type SentryConfiguration struct {
Dsn string `json:"dsn,omitempty"`
}
// SplunkConfiguration defines the configuration options for the Splunk logging system
type SplunkConfiguration struct {
Dsn string `json:"dsn,omitempty"`
Token string `json:"token,omitempty"`
Index string `json:"index,omitempty"`
SendLogs bool `json:"sendLogs"`
}
var rootCmd = &cobra.Command{
Use: "piper",
Short: "Executes CI/CD steps from project 'Piper' ",
Long: `
This project 'Piper' binary provides a CI/CD step library.
It contains many steps which can be used within CI/CD systems as well as directly on e.g. a developer's machine.
`,
}
// GeneralConfig contains global configuration flags for piper binary
var GeneralConfig GeneralConfigOptions
// Execute is the starting point of the piper command line tool
func Execute() {
rootCmd.AddCommand(ArtifactPrepareVersionCommand())
rootCmd.AddCommand(ConfigCommand())
rootCmd.AddCommand(ContainerSaveImageCommand())
rootCmd.AddCommand(CommandLineCompletionCommand())
2019-10-31 14:57:29 +02:00
rootCmd.AddCommand(VersionCommand())
rootCmd.AddCommand(DetectExecuteScanCommand())
refactor(hadolint): implement step in GO (#1169) * initial commit of yaml file * initial commit for HaDoLint in GO * add helper function to load file from url * load config file * write report information to disk * comment the code * refactor groovy code * remove download function from FileUtils * use http.Downloader * rename step files * update generated files * update generated files * remove duplicate commands * add credentials for config url * add generated test file * reuse piperExecuteBin functions * correct step name * update go step * deactivate test * fix import * use differing go step name * rename step * correct result publishing * correct command name * expose tls insecure flag * hand through error * disable tls verification * fix tls disabling * use credentials * mow * reformat * add qgate only if set * correct report name * remove old defaults * add qgate to defaults * handle report name * restore default * remove unused step config * use piperExecuteBin * remove obsolete type * add test cases * remove groovy tests * move client parameter handling to run function * use custom interfaces and mockery * remove commented code * correct struct names * rename parameter dockerfile * add further asserts * cleanup * change file permission to read/write * remove tokenize * add further comments * init http client only if necessary * add todo * Revert "rename parameter dockerfile" This reverts commit 2a570685b89317d20217217894d68242d4620031. * add alias for dockerfile parameter * correct test case * Apply suggestions from code review Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com> * add comment about mock assertions Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com>
2020-11-16 16:14:54 +02:00
rootCmd.AddCommand(HadolintExecuteCommand())
2019-10-30 10:20:25 +02:00
rootCmd.AddCommand(KarmaExecuteTestsCommand())
feat(uiVeri5): golang implmementation for uiVeri5ExecuteTests (#2394) * added uiVeri5ExecuteTests step files * added confPath an regenerated step * added test for uiVeri5ExecuteTests * config modified * added groovy wrapper * ambiguous method fixed * uiveri5 wrapper * removed install command * fixed defaults * added testOptions as confPath arg * test set env * test npm install local * changed env settings * tests regenerated * go generate * fix code climate * overwrite groovy step * remove groovy wrapper go * unstash piper bin * test older node version * test piperExecuteBin * wip * wip * wip * wip * wip * wip * wip * refactored params * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * set testServerUrl as os env for uiveri5 * Update config.yml * fix naming of testServerUrl param * wip * refactored setEnv and fixed tests * wip * step param for NPM_CONFIG_PREFIX * fix runCommand * refactored step param, regenerate, docu, fix tests * fix groovy wrapper test * cleanup * add to CommonStepsTest field whitelist * fixed default pipeline environment vars * fix []string default * fix metadata.go bug * added test for docu metadata gen * fix metadata_test.go in doc gen * Update metadata_generated.go * Update metadata_generated.go * remove npm config prefix param; doc fix * remove tab * changed npm config prefix * removed groovy wrapper test * removed groovy step defaults * modify path variable * modified npm config prefix * fix error wrapper and tests * doc update * add testRepository support * wip * fix testRepository param * wip * add utils * init stash content * wip * wip * wip * add comment for deprecated parameters * fixed commonStepTest * fixed commonStepTest * added error category for testOptions failure * Update vars/uiVeri5ExecuteTests.groovy Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Update vars/uiVeri5ExecuteTests.groovy Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * fix condition Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2021-01-28 12:31:24 +02:00
rootCmd.AddCommand(UiVeri5ExecuteTestsCommand())
rootCmd.AddCommand(SonarExecuteScanCommand())
rootCmd.AddCommand(KubernetesDeployCommand())
rootCmd.AddCommand(XsDeployCommand())
rootCmd.AddCommand(GithubCheckBranchProtectionCommand())
rootCmd.AddCommand(GithubCommentIssueCommand())
rootCmd.AddCommand(GithubCreateIssueCommand())
rootCmd.AddCommand(GithubCreatePullRequestCommand())
rootCmd.AddCommand(GithubPublishReleaseCommand())
rootCmd.AddCommand(GithubSetCommitStatusCommand())
feat(Gitops): new step to update deployment (#2178) * kanikoExecute: improve user experience * ensure proper tags * update permissions in case a container runs with a different user we need to make sure that the orchestrator user can work on the file * update permissions * ensure availablility of directories on Jenkins * (fix) clean up tmp dir in test * add resilience for incorrect step yaml * incorporate PR feedback * Adds piper step to update deployment configuration in external git repository. https://github.wdf.sap.corp/ContinuousDelivery/piper-ita/issues/21 * Adds handling of branchName as an optional parameter * Update resources/metadata/gitopsUpdateDeployment.yaml Feedback about description Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Adapt to interface guide * Refactors to GitopsExecRunner * Refactors to GitopsExecRunner in test * Removes unnecessary mocked methods * Adds tests for git utils * Adds new step to CommonStepsTest.groovy * Updates description from yaml * Restricts visibility of methods and interfaces Adds comments where necessary * Updates comments * Fixes URL name * updates description * updates generated file * Fixes compile issue in CommonStepsTest.groovy * Updates long description * Updates test to run green on all kind of OS * Removes global variables from tests * Default branch: master Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> * Typo in Hierarchy Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> * Refactors test to allow parallel execution * Renames utility variable in gitopsUpdateDeployment.go * Renames error variables in gitopsUpdateDeployment.go * simplified parameters for kubectl * Refactors util classes to use parameters rather than global variables * makes username and password mandatory * remove unnecessary mandatory flag * remove new methods from mock that are not necessary * replaces with EqualError * replaces with NoError * update generated file * refactor tests * refactor tests * make tests parallel executable * parallel execution of tests * Refactors interfaces to stop exposing interfaces * Feedback from PR * Simplifies failing mocks * Renames variables and interfaces * Fixes error messages * shorten variable names * Renames unused parameters in tests * Cleanup nil parameters * Typo * Wrap errors and remove unnecessary logs * Remove containername and filePath from GENERAL scope * correct generated file * corrects expected error messages Co-authored-by: OliverNocon <oliver.nocon@sap.com> Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-10-20 09:05:17 +02:00
rootCmd.AddCommand(GitopsUpdateDeploymentCommand())
rootCmd.AddCommand(CloudFoundryDeleteServiceCommand())
rootCmd.AddCommand(AbapEnvironmentPullGitRepoCommand())
rootCmd.AddCommand(AbapEnvironmentCloneGitRepoCommand())
Add step abapEnvironmentCheckoutBranch (#1832) * Add abaputils pkg and go files * Add ReadServiceKeyAbapEnvironment function * Fixes * Add structs for SC, Pull and Branch * Minor Improvements * Adapt unit tests to new abaputils pkg * Fixes * Add adapted tests * Fixes * Fix cloudfoundry test * Add check for host prefix (HTTPS) * Fix tests + cleanup * Fixes * Fixes * Fix * Add mock for abaputils pkg unit tests * Adapt abaputils comments * Add abapEnvironmentCheckoutBranch step setup * Change description of abapEnvCheckoutBranch step * Add http client code * Disable code due to missing interace * Add coding for use of abaputils * Adapt checkout branch step * Adapt URL for checkout_branch function import * Fixes * Add unit test for missing params case * Fix for missing mapping of CfSpace * Fix for missing mapping of CfSpace * Add working code for a Branch Checkout * Fix host schema * Remove LogoutOption param of unit tests and steps * Fix unit test * Fix unit test CF ReadServiceKey * Add abapEnvironmentCheckoutBranch step setup * Change description of abapEnvCheckoutBranch step * Add http client code * Disable code due to missing interace * Add coding for use of abaputils * Adapt checkout branch step * Adapt URL for checkout_branch function import * Fixes * Fix for missing mapping of CfSpace * Add working code for a Branch Checkout * Adapt changes of abautils pkg * Add test for polling * Minor fix * Fix yaml spacing * Add longdescription to yaml * Refactor abaputil methods * Refactoring * Refactoring * Minor fix * Minor fixeds * Adapt to new abaputils.AUtilsMock * Delete obsolete initial checks for params * Fix manageGitRepoUtils_test.go * Adjust pollEntity tests Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com>
2020-08-04 17:52:28 +02:00
rootCmd.AddCommand(AbapEnvironmentCheckoutBranchCommand())
rootCmd.AddCommand(AbapEnvironmentCreateSystemCommand())
rootCmd.AddCommand(CheckmarxExecuteScanCommand())
rootCmd.AddCommand(FortifyExecuteScanCommand())
rootCmd.AddCommand(MtaBuildCommand())
rootCmd.AddCommand(ProtecodeExecuteScanCommand())
2020-02-28 14:09:46 +02:00
rootCmd.AddCommand(MavenExecuteCommand())
rootCmd.AddCommand(CloudFoundryCreateServiceKeyCommand())
2020-03-13 14:32:37 +02:00
rootCmd.AddCommand(MavenBuildCommand())
rootCmd.AddCommand(MavenExecuteIntegrationCommand())
rootCmd.AddCommand(MavenExecuteStaticCodeChecksCommand())
rootCmd.AddCommand(NexusUploadCommand())
Add abapEnvironmentRunATCCheck step (#1454) * Minor changes * Changed groovy file * Changed generated file * Changed yaml with container config * Changed groovy config * minor changes * minor changes * Changed yaml with aliases * minor changes * minor changes * minor changes * minor changes * minor changes * minor changes * minor changes * minor changes * minor changes * minor changes * Changed yaml aliases * Adapted naming conventions * Removed error code at the end * Adapted configuration * Minor changes * Minor changes * Minor changes * Removed spaces * Removed docker-related config from groovy file * Minor changes * Minor changes * Removed container config * Corrected testing function name * Deleted unnecessary parts * Changed service deletion message * Changed service deletion message * Logging out before throwing error service deletion step fails * Minor changes * Minor changes * Minor changes * Delete .DS_Store * Delete .DS_Store * Delete .DS_Store * Delete .DS_Store * Minor changes * Minor changes * Minor changes * Added newline at end of file * Added newline at end of file * Changes for Pull request optimization * added documentaion * Adapted documentation * Adapted documentation * Adapted documentation * Adapted documentation * Adapted documentation * Added CFDeleteServiceKeys * Added ServiceKey deletion tests * added cfServiceKeys flag explanation to documentation * removed trailing spaces from documentation * resolving conflicts * Changed deletion message an variable naming * Changed tests * Changed tests * Changed tests * Changed tests * Changed CloudFoundryDeleteServiceOptions to options * Changed CloudFoundryDeleteServiceOptions to options * Minor changes * Minor changes * Changed variable naming * Changed error handling * Changed error handling and logging * Changed documentation * Simplified code * Fixed CodeClimate issues * Changed from returning err to nil where no errur returned needed * Add cloudFoundryCreateServiceKey Go Step * Changed Groovy File * Changed aliases * Removed unneccessary parts * Minor changes * Minor changes * Adapted documentation * Adapted tests * Adapted Groovy File * Changed Groovy file * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Removed Groovy Tests for cfCreateServiceKey * Minor changes * Added ATC Check YAML * Added ATC Check generated files * Added test class * Added abapEnvironmentRunATCCheck * Minor changes * Minor changes * Changed groovy * Minor changes * Changed groovy * Changed groovy * Minor changes * Adapted Groovy imports * Adapted Groovy imports * Adapted Groovy imports * Adapted Groovy * Getting ATC results * Changed error message * changed groovy * removed trailing spaces * Added login check * Minor changes * Added step to whitelistScriptReference * Added ATC error message handling * Added groovy file * Added step to groovy tests * corrected metadata file * Debugging * Debugging * Added yaml config parameter for ATC run * Adapted file location of ATC run config to jenkins specific location * Implementing universal pipeline logic for finding yaml config regardless of pipeline * Changed error handling for reading config yaml file * Changed atcrunconfig alias * minor changes * Minor changes * Minor changes * Changed back to dynamic file reading * Minor changes * filepath changes * Removing CF Login * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Removed whitespaces * Added CF functions unit tests * Added invalid parameter handling * Removed package and SC flag * Minor changes * Changed tests * Changed tests * Changed tests * Minor changes * Changed tests * removed unnecessary logout * Added documentation * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Removed trailing spaces * Added newline at end of file * code climate fixes * code climate fixes * code climate fixes * Minor changes * Minor changes * Minor changes * Changed tests * Test changes * Splitted Cloud Foundry functions into two classes * Removed two steps from whtielistScriptReference * removed atcrunConfig alias * issue fixes * Changed docu * Changed docu * Changed docu * Removed trailing spaced from docu * Changed docu * Go generator run * Issue fixes * Remove unnecessary imports Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Update whitelistScript Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Adding piperutils for writing xml file * Persisting ATC Results with piperutils * Set failonMissingReports to true * Refactoring for CodeClimate * Changed result file name * Changed credentials aliases * changing secret name * Removing trailing spaces * Added secret name and alias to docu Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-05-13 14:51:48 +02:00
rootCmd.AddCommand(AbapEnvironmentRunATCCheckCommand())
rootCmd.AddCommand(NpmExecuteScriptsCommand())
rootCmd.AddCommand(NpmExecuteLintCommand())
rootCmd.AddCommand(GctsCreateRepositoryCommand())
rootCmd.AddCommand(GctsExecuteABAPUnitTestsCommand())
rootCmd.AddCommand(GctsDeployCommand())
rootCmd.AddCommand(MalwareExecuteScanCommand())
Implementing cfCreateService in Golang (#1649) * Adapted documentation * Adapted documentation * Adapted documentation * Adapted documentation * Adapted documentation * Added CFDeleteServiceKeys * Added ServiceKey deletion tests * added cfServiceKeys flag explanation to documentation * removed trailing spaces from documentation * resolving conflicts * Changed deletion message an variable naming * Changed tests * Changed tests * Changed tests * Changed tests * Changed CloudFoundryDeleteServiceOptions to options * Changed CloudFoundryDeleteServiceOptions to options * Minor changes * Minor changes * Changed variable naming * Changed error handling * Changed error handling and logging * Changed documentation * Simplified code * Fixed CodeClimate issues * Changed from returning err to nil where no errur returned needed * Add cloudFoundryCreateServiceKey Go Step * Changed Groovy File * Changed aliases * Removed unneccessary parts * Minor changes * Minor changes * Adapted documentation * Adapted tests * Adapted Groovy File * Changed Groovy file * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Removed Groovy Tests for cfCreateServiceKey * Minor changes * Added ATC Check YAML * Added ATC Check generated files * Added test class * Added abapEnvironmentRunATCCheck * Minor changes * Minor changes * Changed groovy * Minor changes * Changed groovy * Changed groovy * Minor changes * Adapted Groovy imports * Adapted Groovy imports * Adapted Groovy imports * Adapted Groovy * Getting ATC results * Changed error message * changed groovy * removed trailing spaces * Added login check * Minor changes * Added step to whitelistScriptReference * Added ATC error message handling * Added groovy file * Added step to groovy tests * corrected metadata file * Debugging * Debugging * Added yaml config parameter for ATC run * Adapted file location of ATC run config to jenkins specific location * Implementing universal pipeline logic for finding yaml config regardless of pipeline * Changed error handling for reading config yaml file * Changed atcrunconfig alias * minor changes * Minor changes * Minor changes * Changed back to dynamic file reading * Minor changes * filepath changes * Removing CF Login * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Minor changes * Removed whitespaces * Added CF functions unit tests * Added invalid parameter handling * Removed package and SC flag * Minor changes * Changed tests * Changed tests * Changed tests * Minor changes * Changed tests * removed unnecessary logout * Added documentation * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Changed docu * Removed trailing spaces * Added newline at end of file * code climate fixes * code climate fixes * code climate fixes * Minor changes * Minor changes * Minor changes * Changed tests * Test changes * Splitted Cloud Foundry functions into two classes * Removed two steps from whtielistScriptReference * removed atcrunConfig alias * issue fixes * Changed docu * Changed docu * Changed docu * Removed trailing spaced from docu * Changed docu * Go generator run * Issue fixes * Remove unnecessary imports Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Update whitelistScript Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Adding piperutils for writing xml file * Persisting ATC Results with piperutils * Set failonMissingReports to true * Refactoring for CodeClimate * Changed result file name * Changed credentials aliases * changing secret name * Removing trailing spaces * Added secret name and alias to docu * PR commit * Go generator * Code Climate fixes * Code Climate fixes * Code Climate fixes * Removed existing groovy tests * Added cfCreateService to fieldRelatedWhiteList * Remarks * added file checking * tests adapted * Changed to execRunner * Removed workingDir definition * Removed workingDir definition * Removed workingDir definition * Added default * Added aliases * Changing to CFUtils Exec Runner * Change defaults * Removed trailing spaces * Refactoring and test changes * Added manifest file default & re-arranged defer func * TestFiles creation added * Changed alias values * Corrected defer logout err return Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com>
2020-08-19 10:17:27 +02:00
rootCmd.AddCommand(CloudFoundryCreateServiceCommand())
rootCmd.AddCommand(CloudFoundryDeployCommand())
rootCmd.AddCommand(GctsRollbackCommand())
Whitesource scan (MVP) (#1658) * Whitesource MVP for Gradle, Golang, and NPM/Yarn * Refactoring * Refactor and cleanup, better error checking * publish stepResults, use pkg/versioning, bubble up errors, add gomod versioning support * Run gofmt and cleanup comments * Resolve PR comments * Update resources/metadata/whitesource.yaml Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> * Only determine project coordinates if they are missing Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com> * Gradle versioning artifact * fix gradle artifact version regexp and refactor * Fix token extraction from output buffer * Fix some issues with pip and jsonfile versioning logic * Remove useless spacing * Remove unnecessary test file and fix naming style for JSONDescriptor * Automatically download wss-unified-agent if file does not exist * adds downloadVulnerabilityReport, checkSecurityViolations, minor refactoring * adds config.ReportDirectoryName, improves readability * Version-wide reporting for vulnerabilities and list of libraries. * Refactor and improve build accuracy * fix sed command * Add includes file pattern config option * Adds --exclude command line flag * run go mod tidy and regenerate step framework * Fix unit tests * revert changes * poll project status before downloading reports * merge with master * go mod tidy, go fmt, and fix whitesource unit test * sync go.mod * sync go.mod again Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Stephan Aßmus <stephan.assmus@sap.com> Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2020-07-01 07:54:13 +02:00
rootCmd.AddCommand(WhitesourceExecuteScanCommand())
rootCmd.AddCommand(GctsCloneRepositoryCommand())
rootCmd.AddCommand(JsonApplyPatchCommand())
rootCmd.AddCommand(KanikoExecuteCommand())
rootCmd.AddCommand(CnbBuildCommand())
Generic build step (#3323) * new step abapEnvironmentBuild * Update piper.go * Update abapEnvironmentBuild.go * update yaml file * Logging for debugging * Update abaputils.go * Update connector.go * assigning connector * delete debugging logging * Update abapEnvironmentBuild.go * certificate to yaml * Update abapEnvironmentBuild.go * add scope * Update abapEnvironmentBuild.go * Update abapEnvironmentBuild.yaml * change certificate name in yaml * test my new gitscript * logging for debugging * debugging... * adding options to client. * skip verification * debugging * debugging... * switch of transportskipverification * changing connector return * deleting additional set options * fixed timeout error * adding certificate * testing without certificate set * testing with certificate set * download, publish and value logic * write values to cpe * logging * adding condition on string length * change publishmethod and some logging * change download method -> using references * evaluation of parameter for download * add case for empty string * adding unittests * Update mockClient.go * make abapEnvironmentBuildUtilsBundle powerful * refactor abapEnvironmentBuild into pieces * check error message * check error message 2 * check error message 3 * check error message 4 * remove check error message * cleanup * adding unittests * unittests and docu * docu * docu * Update abapEnvironmentBuild.md * removing trailing spaces and adding empty lines in docu * Update abapEnvironmentBuild.md * fixing unittest and PR recommen * Update abapEnvironmentPipelineStageBuild.groovy * Update abapEnvironmentPipelineStageBuild.groovy * Update abapEnvironmentPipelineStageBuild.groovy * Update abapEnvironmentPipelineStageBuild.groovy * changes derived from pull request Co-authored-by: tiloKo <70266685+tiloKo@users.noreply.github.com>
2021-12-06 15:43:37 +02:00
rootCmd.AddCommand(AbapEnvironmentBuildCommand())
New step: abapEnvironmentAssemblePackages (#1884) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-08-27 07:54:03 +02:00
rootCmd.AddCommand(AbapEnvironmentAssemblePackagesCommand())
AAKaaS:1 check Component Versions (#2023) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order * AAKaaS CheckCV step * AAKaaS CheckCV step #2 * AAKaaS CheckCV step #3 * AAKaaS CheckCV step #4 * AAKaaS CheckCV step #5 * AAKaaS CheckCV step #6 * AAKaaS CheckCV step #7 * AAKaaS CheckCV step #8 * AAKaaS CheckCV step #9 * AAKaaS CheckCV step #10 * AAKaaS CheckCV step #11 * AAKaaS CheckCV step #12 * AAKaaS CheckCV step #13 Co-authored-by: rosemarieB <45030247+rosemarieB@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-09-17 11:01:19 +02:00
rootCmd.AddCommand(AbapAddonAssemblyKitCheckCVsCommand())
AAKaaS: 2 Check Product Version (#2037) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order * AAKaaS CheckPV step * AAKaaS CheckPV step #2 * AAKaaS CheckPV step #3 * AAKaaS CheckPV step #4 * AAKaaS CheckPV step #5 * Update resources/metadata/abapAddonAssemblyKitCheckCVs.yaml Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> * Update resources/metadata/abapAddonAssemblyKitCheckPV.yaml Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> * Update resources/metadata/abapAddonAssemblyKitCheckPV.yaml Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> * Update resources/metadata/abapAddonAssemblyKitCheckCVs.yaml Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> * AAKaaS CheckPV step #6 Co-authored-by: rosemarieB <45030247+rosemarieB@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-09-17 15:56:14 +02:00
rootCmd.AddCommand(AbapAddonAssemblyKitCheckPVCommand())
AAKaaS: 4 create Target Vector (#2041) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order * AAKaaS publishTV * AAKaaS createTV * AAKaaS createTV #2 * AAKaaS createTV #3 Co-authored-by: rosemarieB <45030247+rosemarieB@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-09-18 11:18:51 +02:00
rootCmd.AddCommand(AbapAddonAssemblyKitCreateTargetVectorCommand())
AAKaaS: 3 publish target Vector (#2040) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order * AAKaaS publishTV * AAKaaS 3 Publish TV #2 Co-authored-by: rosemarieB <45030247+rosemarieB@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-09-18 10:24:46 +02:00
rootCmd.AddCommand(AbapAddonAssemblyKitPublishTargetVectorCommand())
AAKaaS: 5 register package (#2042) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order * AAKaaS register package * AAKaaS register package #2 * Update documentation/docs/steps/abapAddonAssemblyKitPublishTargetVector.md Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> * AAKaaS register package #3 Co-authored-by: rosemarieB <45030247+rosemarieB@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-09-18 14:07:42 +02:00
rootCmd.AddCommand(AbapAddonAssemblyKitRegisterPackagesCommand())
AAKaaS: 6 release package (#2043) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order * AAKaaS register package * AAKaaS register package #2 * Update documentation/docs/steps/abapAddonAssemblyKitPublishTargetVector.md Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> * AAKaaS register package #3 * AAKaaS release package * Update abapAddonAssemblyKitReleasePackages.go * AAKaas release package #2 Co-authored-by: rosemarieB <45030247+rosemarieB@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-09-18 16:40:49 +02:00
rootCmd.AddCommand(AbapAddonAssemblyKitReleasePackagesCommand())
AAKaaS: 7 reserve next package (#2046) * adding my steps * messy step * Update abapEnvironmentAssembly.go * clean up * change yaml * corrections * Update cloudFoundryDeploy.go * update * delete simulation step * remove simulate * Update PiperGoUtils.groovy * Update PiperGoUtils.groovy * Update CommonStepsTest.groovy * add docu * Update abapEnvironmentAssembly.md * changes due to PR * Update .gitignore * b * CV list * Update abapEnvironmentAssembly.go * testing with simulation * Update abapEnvironmentAssembly.go * remove simulation * renaming * Update mkdocs.yml * moving service key to yaml and fixing code climate * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * Update abapEnvironmentAssemblePackages.go * change input * Update abapEnvironmentAssemblePackages.go * change json tag * fixed error handling * documentation * Update abapEnvironmentAssemblePackages.md * Update abapEnvironmentAssemblePackages.md * fixing code climate issues * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * fixing code climate issues * Update abapEnvironmentAssemblePackages.yaml * adding unittests * adding unittests and improved logging * yaml -> json * change scope of cfServiceKeyName * correct indentation * Update CommonStepsTest.groovy * maintain correct step order * AAKaaS register package * AAKaaS register package #2 * Update documentation/docs/steps/abapAddonAssemblyKitPublishTargetVector.md Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> * AAKaaS register package #3 * AAKaaS release package * Update abapAddonAssemblyKitReleasePackages.go * AAKaas release package #2 * AAKaaS reserve * AAKaaS reserve #2 Co-authored-by: rosemarieB <45030247+rosemarieB@users.noreply.github.com> Co-authored-by: Daniel Mieg <56156797+DanielMieg@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com>
2020-09-18 17:27:04 +02:00
rootCmd.AddCommand(AbapAddonAssemblyKitReserveNextPackagesCommand())
rootCmd.AddCommand(CloudFoundryCreateSpaceCommand())
rootCmd.AddCommand(CloudFoundryDeleteSpaceCommand())
rootCmd.AddCommand(VaultRotateSecretIdCommand())
rootCmd.AddCommand(IsChangeInDevelopmentCommand())
rootCmd.AddCommand(TransportRequestUploadCTSCommand())
rootCmd.AddCommand(TransportRequestUploadRFCCommand())
feat(newmanExecute): golang implmementation for newmanExecute (#2513) * Automates first parts of newmanExecute.groovy Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Adds newman installation Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Removes warning Signed-off-by: Fabian Reh <fabian.reh@sap.com> * makes tests robust for later shell calls Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Adds version logging Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Adds tests for version logging Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Adds newman shell execution Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Prepare cloud foundry apps with secrets handling Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Adds further process to CF Utils Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Fixes unit test Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Adds error category Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Add fix to execute step locally Currently only tested on windows machine locally in powershell. Signed-off-by: Fabian Reh <fabian.reh@sap.com> * Adapt unit test to fix of runCommand Signed-off-by: Fabian Reh <fabian.reh@sap.com> * refactored golang step to newmanExecute * wip * added test config * refactored newmanExecute groovy wrapper step * exclude newmanExecute from common step test * cleaups * add credential support * fix groovy credential providing * add import * add stageName * define script * remove unused vars * add import * fix iterator ref * golang secret handling and cleanups * wip * wip * wip * update go step * implement cf credential proposal * testRepository functionality implemented * register secrets to logger * add missing dependecies * test xsuaa credential handling * wip * wip * cleanups * add import * remove mandatory params * add container definition * test runCommand * test runCommand * fix npm path * fix npm path * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * added newmanEnvironment to templating * wip * use env and globals params in runCommand when no templating * fix condition * wip * reverted config edit * updated documentation * install with shell * wip * wip * fix tests * refactor tests * wip * remove old test * wip * escape go tmpl * escape go tmpl * fix defaults * add doc comment * remove test case * refactored newman commands * add cli reporter * refactor options * mock os getenv and fix all tests * refactoring and doc update * go generate * small refactor * spelling * fix newman doc * remove MaskPasswords wrapper; fix stash bug; * docu fix Co-authored-by: Fabian Reh <fabian.reh@sap.com>
2021-03-17 09:08:33 +02:00
rootCmd.AddCommand(NewmanExecuteCommand())
rootCmd.AddCommand(IntegrationArtifactDeployCommand())
rootCmd.AddCommand(TransportRequestUploadSOLMANCommand())
rootCmd.AddCommand(IntegrationArtifactUpdateConfigurationCommand())
rootCmd.AddCommand(IntegrationArtifactGetMplStatusCommand())
rootCmd.AddCommand(IntegrationArtifactGetServiceEndpointCommand())
rootCmd.AddCommand(IntegrationArtifactDownloadCommand())
rootCmd.AddCommand(AbapEnvironmentAssembleConfirmCommand())
rootCmd.AddCommand(IntegrationArtifactUploadCommand())
rootCmd.AddCommand(IntegrationArtifactTriggerIntegrationTestCommand())
rootCmd.AddCommand(IntegrationArtifactUnDeployCommand())
rootCmd.AddCommand(IntegrationArtifactResourceCommand())
rootCmd.AddCommand(TerraformExecuteCommand())
rootCmd.AddCommand(ContainerExecuteStructureTestsCommand())
rootCmd.AddCommand(GaugeExecuteTestsCommand())
rootCmd.AddCommand(BatsExecuteTestsCommand())
rootCmd.AddCommand(PipelineCreateScanSummaryCommand())
rootCmd.AddCommand(TransportRequestDocIDFromGitCommand())
rootCmd.AddCommand(TransportRequestReqIDFromGitCommand())
rootCmd.AddCommand(WritePipelineEnv())
rootCmd.AddCommand(ReadPipelineEnv())
rootCmd.AddCommand(InfluxWriteDataCommand())
rootCmd.AddCommand(AbapEnvironmentRunAUnitTestCommand())
rootCmd.AddCommand(CheckStepActiveCommand())
rootCmd.AddCommand(GolangBuildCommand())
rootCmd.AddCommand(ShellExecuteCommand())
rootCmd.AddCommand(ApiProxyDownloadCommand())
rootCmd.AddCommand(ApiKeyValueMapDownloadCommand())
addRootFlags(rootCmd)
if err := rootCmd.Execute(); err != nil {
log.SetErrorCategory(log.ErrorConfiguration)
log.Entry().WithError(err).Fatal("configuration error")
}
}
func addRootFlags(rootCmd *cobra.Command) {
Changes for Pipeline Reporting (#3213) * Adds GetLog() function to orchestrator * Fixes BUILD_NUMBER env variable * Fixes correct env var for JENKINS_HOME * Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation * Adds possibility to read errorJsons; updates splunk package for log files (WIP) * Uncommenting dev code * Adds GetLog() function to orchestrator * Fixes BUILD_NUMBER env variable * Fixes correct env var for JENKINS_HOME * Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation * Adds possibility to read errorJsons; updates splunk package for log files (WIP) * Uncommenting dev code * Adds GetRequest function which holds the response in memory (not saved to disk) * Implements GetLog() function for ADO, adds function to read PipelineRuntime * PAT has been revoked * Changes http package, s.t. if password only is required basic auth works too * Adds env variable for azure token, error handling in case of unauthenticated/nil response * Adds logging output in case env variable can not be read and fallback variable needs to be used * Adds usage of environment variables for auth, uses jenkins api * Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function * Adds initaliziation function for orchestrator authetnication * Adds settings struct for orchestrator authentication * Adds function to whole logfile to Splunk * Struct for pipeline related telemetry information * Increase messagebatch size to 10k * Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk * Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation * Changes content type to txt * Send telemetry independent of logfiles, increases amount of messages per file * Adds JobURL for orchestrators and UnknownOrchestrator as fallback * telemetry makes use of orchestrator specific information * Adds orchestrator independent correlationID * Adds custom fields for pipeline status * go fmt * Removes env var test - no env variables are read anymore * Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized * Removes Custom fields from telemetry as these can not be reflected in SWA * Adds custom telemetry information (piperHash,..) to each step telemetry information * Removes falltrough in case no orchestrator has been found * Updates tests for orchestrator package * Adds orchestrator import in generated files * Updates generator files for internal library * Adds orchestrator telemetry information to steps * Updates generated files, fatalHook writes to cpe * Go generate from master, go fmt * Adds Custom Data field LastErrorCode * Removes GetLog() test * Update init_unix.go * Update docker_integration_test_executor.go * Update integration_api_cli_test.go * Reverts go1.17 fmt formatting * Reverts go1.17 fmt formatting * Reverts go1.17 fmt formatting * Renames customTelemetryData to stepTelemetryData * Adjustments to orchestrator-package, cleanup, adds JobName * Adjusts commonPipelineEnvironment path * Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package * Go fmt * Changes path for errorDetails, adds debug information * Removes custom fields from step, adds orchestrator, commithash to baseMetadata * Adjusts tests for telemetry package * Adds tests for orchestrator * Updates generated files, initalization of splunk client only if its available in the config * Fixes typo in helper go * Update pkg/http/downloader.go * Update pkg/http/downloader.go * Update pkg/log/fatalHook.go * Update fatalHook.go * Update pkg/splunk/splunk.go * Update pkg/telemetry/data.go * Adds GetBuildStatus() and GetAPIInformation() to orchestrators * error formatting * Bugfix: dont send telemetry data if disabled, adjusts test * go fmt * Use correct error handling * Update pkg/telemetry/telemetry.go * Fixes telemetry disabled in the tests * Fixes http tests * Log fatal errors to logFile * Adds CustomReportingConfig to hooks * Cleanup comments in splunk package * Adds possibility to send telemetry to custom endpoint * Adds debug output for the payload * Debug output for the payload as a string * Adds test cases for changes in telemetry package * go fmt * Adds generated files for new step * Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17 * Adjusts test for sonarExecuteScan * Adjusts test for sonarExecuteScan * Adds explanation for customreportingConfig * Makes disableing of customSend more obvious * Adds custom step reporting to each step, updates generated files, adjusts helper testdata * fixes unit test wrong usage of logging * Send pipeline data altough there has been no error, adjust test cases * Reverts changes for customReporting * Updates generated files, removes customReporting * Removes writing errorDetails to CPE * Reverts usage of customreporting * go fmt * reverts changes in http_test * reverts changes in http_test * Skips integration cnb test Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
var provider orchestrator.OrchestratorSpecificConfigProviding
var err error
Changes for Pipeline Reporting (#3213) * Adds GetLog() function to orchestrator * Fixes BUILD_NUMBER env variable * Fixes correct env var for JENKINS_HOME * Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation * Adds possibility to read errorJsons; updates splunk package for log files (WIP) * Uncommenting dev code * Adds GetLog() function to orchestrator * Fixes BUILD_NUMBER env variable * Fixes correct env var for JENKINS_HOME * Adds getEnv to read env variables with default value, adds test for jenkins GetLog() implementation * Adds possibility to read errorJsons; updates splunk package for log files (WIP) * Uncommenting dev code * Adds GetRequest function which holds the response in memory (not saved to disk) * Implements GetLog() function for ADO, adds function to read PipelineRuntime * PAT has been revoked * Changes http package, s.t. if password only is required basic auth works too * Adds env variable for azure token, error handling in case of unauthenticated/nil response * Adds logging output in case env variable can not be read and fallback variable needs to be used * Adds usage of environment variables for auth, uses jenkins api * Adds init functionality for orchestrators, updates GetLog() and GetPipelineStartTime() function * Adds initaliziation function for orchestrator authetnication * Adds settings struct for orchestrator authentication * Adds function to whole logfile to Splunk * Struct for pipeline related telemetry information * Increase messagebatch size to 10k * Changes splunk package to a pointer based implementation, updates generated files and corresponding template and tests for splunk * Changes telemetry package to pointer based implementation to have multiple telemetry objects, adjusted tests and splunk implementation * Changes content type to txt * Send telemetry independent of logfiles, increases amount of messages per file * Adds JobURL for orchestrators and UnknownOrchestrator as fallback * telemetry makes use of orchestrator specific information * Adds orchestrator independent correlationID * Adds custom fields for pipeline status * go fmt * Removes env var test - no env variables are read anymore * Use UnknownOrchestratorConfigProvider in case the orchestrator can not be initalized * Removes Custom fields from telemetry as these can not be reflected in SWA * Adds custom telemetry information (piperHash,..) to each step telemetry information * Removes falltrough in case no orchestrator has been found * Updates tests for orchestrator package * Adds orchestrator import in generated files * Updates generator files for internal library * Adds orchestrator telemetry information to steps * Updates generated files, fatalHook writes to cpe * Go generate from master, go fmt * Adds Custom Data field LastErrorCode * Removes GetLog() test * Update init_unix.go * Update docker_integration_test_executor.go * Update integration_api_cli_test.go * Reverts go1.17 fmt formatting * Reverts go1.17 fmt formatting * Reverts go1.17 fmt formatting * Renames customTelemetryData to stepTelemetryData * Adjustments to orchestrator-package, cleanup, adds JobName * Adjusts commonPipelineEnvironment path * Adds pipelineTelemetry struct to telemetry package, removes pipeline telemetry structs from splunk package * Go fmt * Changes path for errorDetails, adds debug information * Removes custom fields from step, adds orchestrator, commithash to baseMetadata * Adjusts tests for telemetry package * Adds tests for orchestrator * Updates generated files, initalization of splunk client only if its available in the config * Fixes typo in helper go * Update pkg/http/downloader.go * Update pkg/http/downloader.go * Update pkg/log/fatalHook.go * Update fatalHook.go * Update pkg/splunk/splunk.go * Update pkg/telemetry/data.go * Adds GetBuildStatus() and GetAPIInformation() to orchestrators * error formatting * Bugfix: dont send telemetry data if disabled, adjusts test * go fmt * Use correct error handling * Update pkg/telemetry/telemetry.go * Fixes telemetry disabled in the tests * Fixes http tests * Log fatal errors to logFile * Adds CustomReportingConfig to hooks * Cleanup comments in splunk package * Adds possibility to send telemetry to custom endpoint * Adds debug output for the payload * Debug output for the payload as a string * Adds test cases for changes in telemetry package * go fmt * Adds generated files for new step * Reverts changes for http tests, causing problems with go1.15, changes need to be applied for newer go version >=1.17 * Adjusts test for sonarExecuteScan * Adjusts test for sonarExecuteScan * Adds explanation for customreportingConfig * Makes disableing of customSend more obvious * Adds custom step reporting to each step, updates generated files, adjusts helper testdata * fixes unit test wrong usage of logging * Send pipeline data altough there has been no error, adjust test cases * Reverts changes for customReporting * Updates generated files, removes customReporting * Removes writing errorDetails to CPE * Reverts usage of customreporting * go fmt * reverts changes in http_test * reverts changes in http_test * Skips integration cnb test Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com>
2021-11-18 18:50:03 +02:00
provider, err = orchestrator.NewOrchestratorSpecificConfigProvider()
if err != nil {
log.Entry().Error(err)
provider = &orchestrator.UnknownOrchestratorConfigProvider{}
}
rootCmd.PersistentFlags().StringVar(&GeneralConfig.CorrelationID, "correlationID", provider.GetBuildUrl(), "ID for unique identification of a pipeline run")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.CustomConfig, "customConfig", ".pipeline/config.yml", "Path to the pipeline configuration file")
rootCmd.PersistentFlags().StringSliceVar(&GeneralConfig.GitHubTokens, "gitHubTokens", AccessTokensFromEnvJSON(os.Getenv("PIPER_gitHubTokens")), "List of entries in form of <hostname>:<token> to allow GitHub token authentication for downloading config / defaults")
rootCmd.PersistentFlags().StringSliceVar(&GeneralConfig.DefaultConfig, "defaultConfig", []string{".pipeline/defaults.yaml"}, "Default configurations, passed as path to yaml file")
rootCmd.PersistentFlags().BoolVar(&GeneralConfig.IgnoreCustomDefaults, "ignoreCustomDefaults", false, "Disables evaluation of the parameter 'customDefaults' in the pipeline configuration file")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.ParametersJSON, "parametersJSON", os.Getenv("PIPER_parametersJSON"), "Parameters to be considered in JSON format")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.EnvRootPath, "envRootPath", ".pipeline", "Root path to Piper pipeline shared environments")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.StageName, "stageName", "", "Name of the stage for which configuration should be included")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.StepConfigJSON, "stepConfigJSON", os.Getenv("PIPER_stepConfigJSON"), "Step configuration in JSON format")
2020-01-29 14:17:54 +02:00
rootCmd.PersistentFlags().BoolVar(&GeneralConfig.NoTelemetry, "noTelemetry", false, "Disables telemetry reporting")
rootCmd.PersistentFlags().BoolVarP(&GeneralConfig.Verbose, "verbose", "v", false, "verbose output")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.LogFormat, "logFormat", "default", "Log format to use. Options: default, timestamp, plain, full.")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultServerURL, "vaultServerUrl", "", "The vault server which should be used to fetch credentials")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultNamespace, "vaultNamespace", "", "The vault namespace which should be used to fetch credentials")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultPath, "vaultPath", "", "The path which should be used to fetch credentials")
feat(gcs): allow upload to gcs from steps (#3034) * Upload reports to Google Cloud Storage bucket * Added tests. Made fixes * Update step generation. GCS client was moved to GeneralConfig * Code was refactored * Fixed issues * Fixed issues * Code correction due to PR comments * Improved gcs client and integration tests * Integrated gcp config. Updated step metadata * Fixed issues. Added tests * Added cpe, vault, aliases resolving for reporting parameters * Added tests * Uncommented DeferExitHandler. Removed useless comments * fixed cloning of config * Added comments for exported functions. Removed unused mock * minor fix * Implemented setting of report name via paramRef * some refactoring. Writing tests * Update pkg/config/reporting.go * Update cmd/sonarExecuteScan_generated.go * Apply suggestions from code review * Update pkg/config/reporting.go * Update pkg/config/reporting.go * fixed removing valut secret files * Update pkg/config/reporting.go * restore order * restore order * Apply suggestions from code review * go generate * fixed tests * Update resources/metadata/sonarExecuteScan.yaml * Update resources.go * Fixed tests. Code was regenerated * changed somewhere gcp to gcs. Fixed one test * move gcsSubFolder to input parameters * fixed removing valut secret files * minor fix in integration tests * fix integration tests Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
2021-12-15 16:07:47 +02:00
rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCPJsonKeyFilePath, "gcpJsonKeyFilePath", "", "File path to Google Cloud Platform JSON key file")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCSFolderPath, "gcsFolderPath", "", "GCS folder path. One of the components of GCS target folder")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCSBucketId, "gcsBucketId", "", "Bucket name for Google Cloud Storage")
rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCSSubFolder, "gcsSubFolder", "", "Used to logically separate results of the same step result type")
}
// ResolveAccessTokens reads a list of tokens in format host:token passed via command line
// and transfers this into a map as a more consumable format.
func ResolveAccessTokens(tokenList []string) map[string]string {
tokenMap := map[string]string{}
for _, tokenEntry := range tokenList {
log.Entry().Debugf("processing token %v", tokenEntry)
parts := strings.Split(tokenEntry, ":")
if len(parts) != 2 {
log.Entry().Warningf("wrong format for access token %v", tokenEntry)
} else {
tokenMap[parts[0]] = parts[1]
}
}
return tokenMap
}
// AccessTokensFromEnvJSON resolves access tokens when passed as JSON in an environment variable
func AccessTokensFromEnvJSON(env string) []string {
accessTokens := []string{}
if len(env) == 0 {
return accessTokens
}
err := json.Unmarshal([]byte(env), &accessTokens)
if err != nil {
log.Entry().Infof("Token json '%v' has wrong format.", env)
}
return accessTokens
}
// initStageName initializes GeneralConfig.StageName from either GeneralConfig.ParametersJSON
// or the environment variable (orchestrator specific), unless it has been provided as command line option.
// Log output needs to be suppressed via outputToLog by the getConfig step.
func initStageName(outputToLog bool) {
var stageNameSource string
if outputToLog {
defer func() {
log.Entry().Infof("Using stageName '%s' from %s", GeneralConfig.StageName, stageNameSource)
}()
}
if GeneralConfig.StageName != "" {
// Means it was given as command line argument and has the highest precedence
stageNameSource = "command line arguments"
return
}
// Use stageName from ENV as fall-back, for when extracting it from parametersJSON fails below
provider, err := orchestrator.NewOrchestratorSpecificConfigProvider()
if err != nil {
log.Entry().WithError(err).Warning("Cannot infer stage name from CI environment")
} else {
stageNameSource = "env variable"
GeneralConfig.StageName = provider.GetStageName()
}
if len(GeneralConfig.ParametersJSON) == 0 {
return
}
var params map[string]interface{}
err = json.Unmarshal([]byte(GeneralConfig.ParametersJSON), &params)
if err != nil {
if outputToLog {
log.Entry().Infof("Failed to extract 'stageName' from parametersJSON: %v", err)
}
return
}
stageName, hasKey := params["stageName"]
if !hasKey {
return
}
if stageNameString, ok := stageName.(string); ok && stageNameString != "" {
stageNameSource = "parametersJSON"
GeneralConfig.StageName = stageNameString
}
}
// PrepareConfig reads step configuration from various sources and merges it (defaults, config file, flags, ...)
func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName string, options interface{}, openFile func(s string, t map[string]string) (io.ReadCloser, error)) error {
log.SetFormatter(GeneralConfig.LogFormat)
initStageName(true)
filters := metadata.GetParameterFilters()
2020-01-29 14:17:54 +02:00
// add telemetry parameter "collectTelemetryData" to ALL, GENERAL and PARAMETER filters
filters.All = append(filters.All, "collectTelemetryData")
filters.General = append(filters.General, "collectTelemetryData")
filters.Parameters = append(filters.Parameters, "collectTelemetryData")
feat(gcs): allow upload to gcs from steps (#3034) * Upload reports to Google Cloud Storage bucket * Added tests. Made fixes * Update step generation. GCS client was moved to GeneralConfig * Code was refactored * Fixed issues * Fixed issues * Code correction due to PR comments * Improved gcs client and integration tests * Integrated gcp config. Updated step metadata * Fixed issues. Added tests * Added cpe, vault, aliases resolving for reporting parameters * Added tests * Uncommented DeferExitHandler. Removed useless comments * fixed cloning of config * Added comments for exported functions. Removed unused mock * minor fix * Implemented setting of report name via paramRef * some refactoring. Writing tests * Update pkg/config/reporting.go * Update cmd/sonarExecuteScan_generated.go * Apply suggestions from code review * Update pkg/config/reporting.go * Update pkg/config/reporting.go * fixed removing valut secret files * Update pkg/config/reporting.go * restore order * restore order * Apply suggestions from code review * go generate * fixed tests * Update resources/metadata/sonarExecuteScan.yaml * Update resources.go * Fixed tests. Code was regenerated * changed somewhere gcp to gcs. Fixed one test * move gcsSubFolder to input parameters * fixed removing valut secret files * minor fix in integration tests * fix integration tests Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
2021-12-15 16:07:47 +02:00
envParams := metadata.GetResourceParameters(GeneralConfig.EnvRootPath, "commonPipelineEnvironment")
reportingEnvParams := config.ReportingParameters.GetResourceParameters(GeneralConfig.EnvRootPath, "commonPipelineEnvironment")
resourceParams := mergeResourceParameters(envParams, reportingEnvParams)
flagValues := config.AvailableFlagValues(cmd, &filters)
var myConfig config.Config
var stepConfig config.StepConfig
// add vault credentials so that configuration can be fetched from vault
if GeneralConfig.VaultRoleID == "" {
GeneralConfig.VaultRoleID = os.Getenv("PIPER_vaultAppRoleID")
}
if GeneralConfig.VaultRoleSecretID == "" {
GeneralConfig.VaultRoleSecretID = os.Getenv("PIPER_vaultAppRoleSecretID")
}
if GeneralConfig.VaultToken == "" {
GeneralConfig.VaultToken = os.Getenv("PIPER_vaultToken")
}
myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID, GeneralConfig.VaultToken)
if len(GeneralConfig.StepConfigJSON) != 0 {
// ignore config & defaults in favor of passed stepConfigJSON
stepConfig = config.GetStepConfigWithJSON(flagValues, GeneralConfig.StepConfigJSON, filters)
log.Entry().Infof("Project config: passed via JSON")
log.Entry().Infof("Project defaults: passed via JSON")
} else {
// use config & defaults
var customConfig io.ReadCloser
var err error
//accept that config file and defaults cannot be loaded since both are not mandatory here
{
projectConfigFile := getProjectConfigFile(GeneralConfig.CustomConfig)
if exists, err := piperutils.FileExists(projectConfigFile); exists {
log.Entry().Infof("Project config: '%s'", projectConfigFile)
if customConfig, err = openFile(projectConfigFile, GeneralConfig.GitHubAccessTokens); err != nil {
return errors.Wrapf(err, "Cannot read '%s'", projectConfigFile)
}
} else {
log.Entry().Infof("Project config: NONE ('%s' does not exist)", projectConfigFile)
customConfig = nil
}
}
var defaultConfig []io.ReadCloser
if len(GeneralConfig.DefaultConfig) == 0 {
log.Entry().Info("Project defaults: NONE")
}
for _, projectDefaultFile := range GeneralConfig.DefaultConfig {
fc, err := openFile(projectDefaultFile, GeneralConfig.GitHubAccessTokens)
// only create error for non-default values
if err != nil {
if projectDefaultFile != ".pipeline/defaults.yaml" {
log.Entry().Infof("Project defaults: '%s'", projectDefaultFile)
return errors.Wrapf(err, "Cannot read '%s'", projectDefaultFile)
}
} else {
log.Entry().Infof("Project defaults: '%s'", projectDefaultFile)
defaultConfig = append(defaultConfig, fc)
}
}
feat(gcs): allow upload to gcs from steps (#3034) * Upload reports to Google Cloud Storage bucket * Added tests. Made fixes * Update step generation. GCS client was moved to GeneralConfig * Code was refactored * Fixed issues * Fixed issues * Code correction due to PR comments * Improved gcs client and integration tests * Integrated gcp config. Updated step metadata * Fixed issues. Added tests * Added cpe, vault, aliases resolving for reporting parameters * Added tests * Uncommented DeferExitHandler. Removed useless comments * fixed cloning of config * Added comments for exported functions. Removed unused mock * minor fix * Implemented setting of report name via paramRef * some refactoring. Writing tests * Update pkg/config/reporting.go * Update cmd/sonarExecuteScan_generated.go * Apply suggestions from code review * Update pkg/config/reporting.go * Update pkg/config/reporting.go * fixed removing valut secret files * Update pkg/config/reporting.go * restore order * restore order * Apply suggestions from code review * go generate * fixed tests * Update resources/metadata/sonarExecuteScan.yaml * Update resources.go * Fixed tests. Code was regenerated * changed somewhere gcp to gcs. Fixed one test * move gcsSubFolder to input parameters * fixed removing valut secret files * minor fix in integration tests * fix integration tests Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
2021-12-15 16:07:47 +02:00
stepConfig, err = myConfig.GetStepConfig(flagValues, GeneralConfig.ParametersJSON, customConfig, defaultConfig, GeneralConfig.IgnoreCustomDefaults, filters, *metadata, resourceParams, GeneralConfig.StageName, stepName)
if verbose, ok := stepConfig.Config["verbose"].(bool); ok && verbose {
log.SetVerbose(verbose)
GeneralConfig.Verbose = verbose
} else if !ok && stepConfig.Config["verbose"] != nil {
log.Entry().Warnf("invalid value for parameter verbose: '%v'", stepConfig.Config["verbose"])
}
if err != nil {
return errors.Wrap(err, "retrieving step configuration failed")
}
}
2020-01-29 14:17:54 +02:00
if fmt.Sprintf("%v", stepConfig.Config["collectTelemetryData"]) == "false" {
GeneralConfig.NoTelemetry = true
}
stepConfig.Config = checkTypes(stepConfig.Config, options)
confJSON, _ := json.Marshal(stepConfig.Config)
_ = json.Unmarshal(confJSON, &options)
config.MarkFlagsWithValue(cmd, stepConfig)
retrieveHookConfig(stepConfig.HookConfig, &GeneralConfig.HookConfig)
feat(gcs): allow upload to gcs from steps (#3034) * Upload reports to Google Cloud Storage bucket * Added tests. Made fixes * Update step generation. GCS client was moved to GeneralConfig * Code was refactored * Fixed issues * Fixed issues * Code correction due to PR comments * Improved gcs client and integration tests * Integrated gcp config. Updated step metadata * Fixed issues. Added tests * Added cpe, vault, aliases resolving for reporting parameters * Added tests * Uncommented DeferExitHandler. Removed useless comments * fixed cloning of config * Added comments for exported functions. Removed unused mock * minor fix * Implemented setting of report name via paramRef * some refactoring. Writing tests * Update pkg/config/reporting.go * Update cmd/sonarExecuteScan_generated.go * Apply suggestions from code review * Update pkg/config/reporting.go * Update pkg/config/reporting.go * fixed removing valut secret files * Update pkg/config/reporting.go * restore order * restore order * Apply suggestions from code review * go generate * fixed tests * Update resources/metadata/sonarExecuteScan.yaml * Update resources.go * Fixed tests. Code was regenerated * changed somewhere gcp to gcs. Fixed one test * move gcsSubFolder to input parameters * fixed removing valut secret files * minor fix in integration tests * fix integration tests Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
2021-12-15 16:07:47 +02:00
if GeneralConfig.GCPJsonKeyFilePath == "" {
GeneralConfig.GCPJsonKeyFilePath, _ = stepConfig.Config["gcpJsonKeyFilePath"].(string)
}
if GeneralConfig.GCSFolderPath == "" {
GeneralConfig.GCSFolderPath, _ = stepConfig.Config["gcsFolderPath"].(string)
}
if GeneralConfig.GCSBucketId == "" {
GeneralConfig.GCSBucketId, _ = stepConfig.Config["gcsBucketId"].(string)
}
if GeneralConfig.GCSSubFolder == "" {
GeneralConfig.GCSSubFolder, _ = stepConfig.Config["gcsSubFolder"].(string)
}
return nil
}
func retrieveHookConfig(source map[string]interface{}, target *HookConfiguration) {
if source != nil {
log.Entry().Info("Retrieving hook configuration")
b, err := json.Marshal(source)
if err != nil {
log.Entry().Warningf("Failed to marshal source hook configuration: %v", err)
}
err = json.Unmarshal(b, target)
if err != nil {
log.Entry().Warningf("Failed to retrieve hook configuration: %v", err)
}
}
}
var errIncompatibleTypes = fmt.Errorf("incompatible types")
func checkTypes(config map[string]interface{}, options interface{}) map[string]interface{} {
optionsType := getStepOptionsStructType(options)
for paramName := range config {
optionsField := findStructFieldByJSONTag(paramName, optionsType)
if optionsField == nil {
continue
}
if config[paramName] == nil {
// There is a key, but no value. This can result from merging values from the CPE.
continue
}
paramValueType := reflect.ValueOf(config[paramName])
if optionsField.Type.Kind() == paramValueType.Kind() {
// Types already match, nothing to do
continue
}
var typeError error = nil
switch paramValueType.Kind() {
case reflect.String:
typeError = convertValueFromString(config, optionsField, paramName, paramValueType.String())
case reflect.Float32, reflect.Float64:
typeError = convertValueFromFloat(config, optionsField, paramName, paramValueType.Float())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
typeError = convertValueFromInt(config, optionsField, paramName, paramValueType.Int())
default:
log.Entry().Warnf("Config value for '%s' is of unexpected type %s, expected %s. "+
"The value may be ignored as a result. To avoid any risk, specify this value with explicit type.",
paramName, paramValueType.Kind(), optionsField.Type.Kind())
}
if typeError != nil {
typeError = fmt.Errorf("config value for '%s' is of unexpected type %s, expected %s: %w",
paramName, paramValueType.Kind(), optionsField.Type.Kind(), typeError)
log.SetErrorCategory(log.ErrorConfiguration)
log.Entry().WithError(typeError).Fatal("type error in configuration")
}
}
return config
}
func convertValueFromString(config map[string]interface{}, optionsField *reflect.StructField, paramName, paramValue string) error {
switch optionsField.Type.Kind() {
case reflect.Slice, reflect.Array:
// Could do automatic conversion for those types in theory,
// but that might obscure what really happens in error cases.
return fmt.Errorf("expected type to be a list (or slice, or array) but got string")
case reflect.Bool:
// Sensible to convert strings "true"/"false" to respective boolean values as it is
// common practice to write booleans as string in yaml files.
paramValue = strings.ToLower(paramValue)
if paramValue == "true" {
config[paramName] = true
return nil
} else if paramValue == "false" {
config[paramName] = false
return nil
}
}
return errIncompatibleTypes
}
func convertValueFromFloat(config map[string]interface{}, optionsField *reflect.StructField, paramName string, paramValue float64) error {
switch optionsField.Type.Kind() {
case reflect.String:
val := strconv.FormatFloat(paramValue, 'f', -1, 64)
// if Sprinted value and val are equal, we can be pretty sure that the result fits
// for very large numbers for example an exponential format is printed
if val == fmt.Sprint(paramValue) {
config[paramName] = val
return nil
}
// allow float numbers containing a decimal separator
if strings.Contains(val, ".") {
config[paramName] = val
return nil
}
// if now no decimal separator is available we cannot be sure that the result is correct:
// long numbers like e.g. 73554900100200011600 will not be represented correctly after reading the yaml
// thus we cannot assume that the string is correct.
// short numbers will be handled as int anyway
return errIncompatibleTypes
case reflect.Float32:
config[paramName] = float32(paramValue)
return nil
case reflect.Float64:
config[paramName] = paramValue
return nil
case reflect.Int:
// Treat as type-mismatch only in case the conversion would be lossy.
// In that case, the json.Unmarshall() would indeed just drop it, so we want to fail.
if float64(int(paramValue)) == paramValue {
config[paramName] = int(paramValue)
return nil
}
}
return errIncompatibleTypes
}
func convertValueFromInt(config map[string]interface{}, optionsField *reflect.StructField, paramName string, paramValue int64) error {
switch optionsField.Type.Kind() {
case reflect.String:
config[paramName] = strconv.FormatInt(paramValue, 10)
return nil
case reflect.Float32:
config[paramName] = float32(paramValue)
return nil
case reflect.Float64:
config[paramName] = float64(paramValue)
return nil
}
return errIncompatibleTypes
}
func findStructFieldByJSONTag(tagName string, optionsType reflect.Type) *reflect.StructField {
for i := 0; i < optionsType.NumField(); i++ {
field := optionsType.Field(i)
tag := field.Tag.Get("json")
if tagName == tag || tagName+",omitempty" == tag {
return &field
}
}
return nil
}
func getStepOptionsStructType(stepOptions interface{}) reflect.Type {
typedOptions := reflect.ValueOf(stepOptions)
if typedOptions.Kind() == reflect.Ptr {
typedOptions = typedOptions.Elem()
}
return typedOptions.Type()
}
func getProjectConfigFile(name string) string {
var altName string
if ext := filepath.Ext(name); ext == ".yml" {
altName = fmt.Sprintf("%v.yaml", strings.TrimSuffix(name, ext))
} else if ext == "yaml" {
altName = fmt.Sprintf("%v.yml", strings.TrimSuffix(name, ext))
}
fileExists, _ := piperutils.FileExists(name)
altExists, _ := piperutils.FileExists(altName)
// configured filename will always take precedence, even if not existing
if !fileExists && altExists {
return altName
}
return name
}
feat(gcs): allow upload to gcs from steps (#3034) * Upload reports to Google Cloud Storage bucket * Added tests. Made fixes * Update step generation. GCS client was moved to GeneralConfig * Code was refactored * Fixed issues * Fixed issues * Code correction due to PR comments * Improved gcs client and integration tests * Integrated gcp config. Updated step metadata * Fixed issues. Added tests * Added cpe, vault, aliases resolving for reporting parameters * Added tests * Uncommented DeferExitHandler. Removed useless comments * fixed cloning of config * Added comments for exported functions. Removed unused mock * minor fix * Implemented setting of report name via paramRef * some refactoring. Writing tests * Update pkg/config/reporting.go * Update cmd/sonarExecuteScan_generated.go * Apply suggestions from code review * Update pkg/config/reporting.go * Update pkg/config/reporting.go * fixed removing valut secret files * Update pkg/config/reporting.go * restore order * restore order * Apply suggestions from code review * go generate * fixed tests * Update resources/metadata/sonarExecuteScan.yaml * Update resources.go * Fixed tests. Code was regenerated * changed somewhere gcp to gcs. Fixed one test * move gcsSubFolder to input parameters * fixed removing valut secret files * minor fix in integration tests * fix integration tests Co-authored-by: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Co-authored-by: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Co-authored-by: Sven Merk <33895725+nevskrem@users.noreply.github.com>
2021-12-15 16:07:47 +02:00
func mergeResourceParameters(resParams ...map[string]interface{}) map[string]interface{} {
result := make(map[string]interface{})
for _, m := range resParams {
for k, v := range m {
result[k] = v
}
}
return result
}