From b6884832baa9cfdfe776e4ced51993a539c98bb5 Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Wed, 30 Oct 2019 09:20:25 +0100 Subject: [PATCH 01/13] Add karma command --- cmd/piper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/piper.go b/cmd/piper.go index b868a5ecc..70eb63430 100644 --- a/cmd/piper.go +++ b/cmd/piper.go @@ -39,6 +39,7 @@ var generalConfig generalConfigOptions func Execute() { rootCmd.AddCommand(ConfigCommand()) + rootCmd.AddCommand(KarmaExecuteTestsCommand()) addRootFlags(rootCmd) if err := rootCmd.Execute(); err != nil { From 6256a0b9aa4e5d28fecaa601b49b2f707c39e9ee Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Mon, 4 Nov 2019 16:07:30 +0100 Subject: [PATCH 02/13] Update githubPublishRelease --- cmd/githubPublishRelease.go | 148 +++++++++++++++ cmd/githubPublishRelease_generated.go | 177 ++++++++++++++++++ cmd/githubPublishRelease_generated_test.go | 16 ++ cmd/githubPublishRelease_test.go | 38 ++++ cmd/karmaExecuteTests_generated.go | 2 - cmd/piper.go | 1 + cmd/piper_test.go | 5 +- go.mod | 2 + go.sum | 21 +++ pkg/generator/step-metadata.go | 20 +- pkg/generator/step-metadata_test.go | 9 +- .../step_code_generated.golden | 2 +- pkg/github/github.go | 23 +++ resources/metadata/githubrelease.yaml | 132 +++++++++++++ 14 files changed, 579 insertions(+), 17 deletions(-) create mode 100644 cmd/githubPublishRelease.go create mode 100644 cmd/githubPublishRelease_generated.go create mode 100644 cmd/githubPublishRelease_generated_test.go create mode 100644 cmd/githubPublishRelease_test.go create mode 100644 pkg/github/github.go create mode 100644 resources/metadata/githubrelease.yaml diff --git a/cmd/githubPublishRelease.go b/cmd/githubPublishRelease.go new file mode 100644 index 000000000..aa6044303 --- /dev/null +++ b/cmd/githubPublishRelease.go @@ -0,0 +1,148 @@ +package cmd + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-github/v28/github" + "github.com/pkg/errors" + + piperGithub "github.com/SAP/jenkins-library/pkg/github" +) + +type githubRepoClient interface { + GetLatestRelease(ctx context.Context, owner string, repo string) (*github.RepositoryRelease, *github.Response, error) + CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) +} + +type githubIssueClient interface { + ListByRepo(ctx context.Context, owner string, repo string, opt *github.IssueListByRepoOptions) ([]*github.Issue, *github.Response, error) +} + +func githubPublishRelease(myGithubPublishReleaseOptions githubPublishReleaseOptions) error { + ctx, client, err := piperGithub.NewClient(myGithubPublishReleaseOptions.GithubToken, myGithubPublishReleaseOptions.GithubAPIURL, myGithubPublishReleaseOptions.GithubAPIURL) + if err != nil { + return err + } + + err = runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, client.Repositories, client.Issues) + if err != nil { + return err + } + + return nil +} + +func runGithubPublishRelease(ctx context.Context, myGithubPublishReleaseOptions *githubPublishReleaseOptions, ghRepoClient githubRepoClient, ghIssueClient githubIssueClient) error { + + var publishedAt github.Timestamp + lastRelease, resp, err := ghRepoClient.GetLatestRelease(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo) + if err != nil { + if resp.StatusCode == 404 { + //first release + myGithubPublishReleaseOptions.AddDeltaToLastRelease = false + publishedAt = lastRelease.GetPublishedAt() + } else { + return errors.Wrap(err, "Error occured when retrieving latest GitHub releass") + } + } + + releaseBody := "" + + if len(myGithubPublishReleaseOptions.ReleaseBodyHeader) > 0 { + releaseBody += myGithubPublishReleaseOptions.ReleaseBodyHeader + "
" + } + + if myGithubPublishReleaseOptions.AddClosedIssues { + releaseBody += getClosedIssuesText(ctx, publishedAt, myGithubPublishReleaseOptions, ghIssueClient) + } + + if myGithubPublishReleaseOptions.AddDeltaToLastRelease { + releaseBody += getReleaseDeltaText(myGithubPublishReleaseOptions, lastRelease) + } + + release := github.RepositoryRelease{ + TagName: &myGithubPublishReleaseOptions.Version, + TargetCommitish: &myGithubPublishReleaseOptions.Commitish, + Name: &myGithubPublishReleaseOptions.Version, + Body: &releaseBody, + } + + //create release + createdRelease, _, err := ghRepoClient.CreateRelease(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, &release) + if err != nil { + return errors.Wrapf(err, "creation of release '%v' failed", release.TagName) + } + + // todo switch to logging + fmt.Printf("Release %v created on %v/%v", *createdRelease.TagName, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo) + + return nil +} + +func getClosedIssuesText(ctx context.Context, publishedAt github.Timestamp, myGithubPublishReleaseOptions *githubPublishReleaseOptions, ghIssueClient githubIssueClient) string { + closedIssuesText := "" + options := github.IssueListByRepoOptions{ + State: "closed", + Direction: "asc", + Since: publishedAt.Time, + } + if len(myGithubPublishReleaseOptions.Labels) > 0 { + options.Labels = myGithubPublishReleaseOptions.Labels + } + ghIssues, _, err := ghIssueClient.ListByRepo(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, &options) + if err != nil { + //log error + } + + prTexts := []string{"
**List of closed pull-requests since last release**"} + issueTexts := []string{"
**List of closed issues since last release**"} + + for _, issue := range ghIssues { + if issue.IsPullRequest() && !isExcluded(issue, myGithubPublishReleaseOptions.ExcludeLabels) { + prTexts = append(prTexts, fmt.Sprintf("[#%v](%v): %v", issue.GetNumber(), issue.GetHTMLURL(), issue.GetTitle())) + } else if !issue.IsPullRequest() && !isExcluded(issue, myGithubPublishReleaseOptions.ExcludeLabels) { + issueTexts = append(issueTexts, fmt.Sprintf("[#%v](%v): %v", issue.GetNumber(), issue.GetHTMLURL(), issue.GetTitle())) + } + } + + if len(prTexts) > 1 { + closedIssuesText += strings.Join(prTexts, "\n") + "\n" + } + + if len(issueTexts) > 1 { + closedIssuesText += strings.Join(issueTexts, "\n") + "\n" + } + return closedIssuesText +} + +func getReleaseDeltaText(myGithubPublishReleaseOptions *githubPublishReleaseOptions, lastRelease *github.RepositoryRelease) string { + releaseDeltaText := "" + + //add delta link to previous release + releaseDeltaText += "
**Changes**
" + releaseDeltaText += fmt.Sprintf( + "[%v...%v](%v/%v/%v/compare/%v...%v)
", + lastRelease.GetTagName(), + myGithubPublishReleaseOptions.Version, + myGithubPublishReleaseOptions.GithubServerURL, + myGithubPublishReleaseOptions.GithubOrg, + myGithubPublishReleaseOptions.GithubRepo, + lastRelease.GetTagName(), myGithubPublishReleaseOptions.Version, + ) + + return releaseDeltaText +} + +func isExcluded(issue *github.Issue, excludeLabels []string) bool { + //issue.Labels[0].GetName() + for _, ex := range excludeLabels { + for _, l := range issue.Labels { + if ex == l.GetName() { + return true + } + } + } + return false +} diff --git a/cmd/githubPublishRelease_generated.go b/cmd/githubPublishRelease_generated.go new file mode 100644 index 000000000..10cc6e2aa --- /dev/null +++ b/cmd/githubPublishRelease_generated.go @@ -0,0 +1,177 @@ +package cmd + +import ( + "os" + + "github.com/SAP/jenkins-library/pkg/config" + "github.com/spf13/cobra" +) + +type githubPublishReleaseOptions struct { + AddClosedIssues bool `json:"addClosedIssues,omitempty"` + AddDeltaToLastRelease bool `json:"addDeltaToLastRelease,omitempty"` + AssetPath string `json:"assetPath,omitempty"` + Commitish string `json:"commitish,omitempty"` + ExcludeLabels []string `json:"excludeLabels,omitempty"` + GithubAPIURL string `json:"githubApiUrl,omitempty"` + GithubOrg string `json:"githubOrg,omitempty"` + GithubRepo string `json:"githubRepo,omitempty"` + GithubServerURL string `json:"githubServerUrl,omitempty"` + GithubToken string `json:"githubToken,omitempty"` + Labels []string `json:"labels,omitempty"` + ReleaseBodyHeader string `json:"releaseBodyHeader,omitempty"` + Update bool `json:"update,omitempty"` + Version string `json:"version,omitempty"` +} + +var myGithubPublishReleaseOptions githubPublishReleaseOptions +var githubPublishReleaseStepConfigJSON string + +// GithubPublishReleaseCommand Publish a release in GitHub +func GithubPublishReleaseCommand() *cobra.Command { + metadata := githubPublishReleaseMetadata() + var createGithubPublishReleaseCmd = &cobra.Command{ + Use: "githubPublishRelease", + Short: "Publish a release in GitHub", + Long: `This step creates a tag in your GitHub repository together with a release. +The release can be filled with text plus additional information like: + +* Closed pull request since last release +* Closed issues since last release +* Link to delta information showing all commits since last release + +The result looks like + +![Example release](../images/githubRelease.png)`, + PreRunE: func(cmd *cobra.Command, args []string) error { + return PrepareConfig(cmd, &metadata, "githubPublishRelease", &myGithubPublishReleaseOptions, openPiperFile) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return githubPublishRelease(myGithubPublishReleaseOptions) + }, + } + + addGithubPublishReleaseFlags(createGithubPublishReleaseCmd) + return createGithubPublishReleaseCmd +} + +func addGithubPublishReleaseFlags(cmd *cobra.Command) { + cmd.Flags().BoolVar(&myGithubPublishReleaseOptions.AddClosedIssues, "addClosedIssues", false, "If set to `true`, closed issues and merged pull-requests since the last release will added below the `releaseBodyHeader`") + cmd.Flags().BoolVar(&myGithubPublishReleaseOptions.AddDeltaToLastRelease, "addDeltaToLastRelease", false, "If set to `true`, a link will be added to the relese information that brings up all commits since the last release.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.AssetPath, "assetPath", os.Getenv("PIPER_assetPath"), "Path to a release asset which should be uploaded to the list of release assets.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Commitish, "commitish", "master", "Target git commitish for the release") + cmd.Flags().StringSliceVar(&myGithubPublishReleaseOptions.ExcludeLabels, "excludeLabels", []string{}, "Allows to exclude issues with dedicated list of labels.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubAPIURL, "githubApiUrl", "https://api.github.com", "Set the GitHub API url.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubOrg, "githubOrg", os.Getenv("PIPER_githubOrg"), "Set the GitHub organization.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubRepo, "githubRepo", os.Getenv("PIPER_githubRepo"), "Set the GitHub repository.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubServerURL, "githubServerUrl", "https://github.com", "GitHub server url for end-user access.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") + cmd.Flags().StringSliceVar(&myGithubPublishReleaseOptions.Labels, "labels", []string{}, "Labels to include in issue search.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.ReleaseBodyHeader, "releaseBodyHeader", os.Getenv("PIPER_releaseBodyHeader"), "Content which will appear for the release.") + cmd.Flags().BoolVar(&myGithubPublishReleaseOptions.Update, "update", false, "Specify if the release should be updated in case it already exists") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Version, "version", os.Getenv("PIPER_version"), "Define the version number which will be written as tag as well as release name.") + + cmd.MarkFlagRequired("githubApiUrl") + cmd.MarkFlagRequired("githubOrg") + cmd.MarkFlagRequired("githubRepo") + cmd.MarkFlagRequired("githubServerUrl") + cmd.MarkFlagRequired("githubToken") + cmd.MarkFlagRequired("version") +} + +// retrieve step metadata +func githubPublishReleaseMetadata() config.StepData { + var theMetaData = config.StepData{ + Spec: config.StepSpec{ + Inputs: config.StepInputs{ + Parameters: []config.StepParameters{ + { + Name: "addClosedIssues", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + }, + { + Name: "addDeltaToLastRelease", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + }, + { + Name: "assetPath", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + }, + { + Name: "commitish", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + }, + { + Name: "excludeLabels", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + }, + { + Name: "githubApiUrl", + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + }, + { + Name: "githubOrg", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + }, + { + Name: "githubRepo", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + }, + { + Name: "githubServerUrl", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + }, + { + Name: "githubToken", + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + }, + { + Name: "labels", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + }, + { + Name: "releaseBodyHeader", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + }, + { + Name: "update", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "bool", + Mandatory: false, + }, + { + Name: "version", + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + }, + }, + }, + }, + } + return theMetaData +} diff --git a/cmd/githubPublishRelease_generated_test.go b/cmd/githubPublishRelease_generated_test.go new file mode 100644 index 000000000..a17000345 --- /dev/null +++ b/cmd/githubPublishRelease_generated_test.go @@ -0,0 +1,16 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGithubPublishReleaseCommand(t *testing.T) { + + testCmd := GithubPublishReleaseCommand() + + // only high level testing performed - details are tested in step generation procudure + assert.Equal(t, "githubPublishRelease", testCmd.Use, "command name incorrect") + +} diff --git a/cmd/githubPublishRelease_test.go b/cmd/githubPublishRelease_test.go new file mode 100644 index 000000000..33ff34fa3 --- /dev/null +++ b/cmd/githubPublishRelease_test.go @@ -0,0 +1,38 @@ +package cmd + +import ( + "fmt" + "testing" + + "github.com/google/go-github/v28/github" + "github.com/stretchr/testify/assert" +) + +func TestRunGithubPublishRelease(t *testing.T) { + +} + +func TestIsExcluded(t *testing.T) { + + l1 := "label1" + l2 := "label2" + + tt := []struct { + issue *github.Issue + excludeLabels []string + expected bool + }{ + {issue: nil, excludeLabels: nil, expected: false}, + {issue: &github.Issue{}, excludeLabels: nil, expected: false}, + {issue: &github.Issue{Labels: []github.Label{{Name: &l1}}}, excludeLabels: nil, expected: false}, + {issue: &github.Issue{Labels: []github.Label{{Name: &l1}}}, excludeLabels: []string{"label0"}, expected: false}, + {issue: &github.Issue{Labels: []github.Label{{Name: &l1}}}, excludeLabels: []string{"label1"}, expected: true}, + {issue: &github.Issue{Labels: []github.Label{{Name: &l1}, {Name: &l2}}}, excludeLabels: []string{}, expected: false}, + {issue: &github.Issue{Labels: []github.Label{{Name: &l1}, {Name: &l2}}}, excludeLabels: []string{"label1"}, expected: true}, + } + + for k, v := range tt { + assert.Equal(t, v.expected, isExcluded(v.issue, v.excludeLabels), fmt.Sprintf("Run %v failed", k)) + } + +} diff --git a/cmd/karmaExecuteTests_generated.go b/cmd/karmaExecuteTests_generated.go index 5b331d126..919b98915 100644 --- a/cmd/karmaExecuteTests_generated.go +++ b/cmd/karmaExecuteTests_generated.go @@ -1,8 +1,6 @@ package cmd import ( - //"os" - "github.com/SAP/jenkins-library/pkg/config" "github.com/spf13/cobra" ) diff --git a/cmd/piper.go b/cmd/piper.go index 70eb63430..9eb2817cb 100644 --- a/cmd/piper.go +++ b/cmd/piper.go @@ -40,6 +40,7 @@ func Execute() { rootCmd.AddCommand(ConfigCommand()) rootCmd.AddCommand(KarmaExecuteTestsCommand()) + rootCmd.AddCommand(GithubPublishReleaseCommand()) addRootFlags(rootCmd) if err := rootCmd.Execute(); err != nil { diff --git a/cmd/piper_test.go b/cmd/piper_test.go index c824d9a67..72bbbfce6 100644 --- a/cmd/piper_test.go +++ b/cmd/piper_test.go @@ -13,7 +13,6 @@ import ( "github.com/stretchr/testify/assert" ) - type execMockRunner struct { dir []string calls []execCall @@ -42,11 +41,11 @@ func (m *execMockRunner) RunExecutable(e string, p ...string) error { return nil } -func(m *shellMockRunner) Dir(d string) { +func (m *shellMockRunner) Dir(d string) { m.dir = d } -func(m *shellMockRunner) RunShell(s string, c string) error { +func (m *shellMockRunner) RunShell(s string, c string) error { m.calls = append(m.calls, c) return nil } diff --git a/go.mod b/go.mod index 2c87f65d7..12d32a621 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.13 require ( github.com/ghodss/yaml v1.0.0 + github.com/google/go-github/v28 v28.1.1 github.com/pkg/errors v0.8.1 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.2.2 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 ) diff --git a/go.sum b/go.sum index 796e8ee0a..075d71733 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -9,6 +10,12 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= +github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -35,8 +42,22 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/generator/step-metadata.go b/pkg/generator/step-metadata.go index 299e679c7..b170f5903 100644 --- a/pkg/generator/step-metadata.go +++ b/pkg/generator/step-metadata.go @@ -30,7 +30,7 @@ type stepInfo struct { const stepGoTemplate = `package cmd import ( - //"os" + {{if .OSImport}}"os"{{end}} "github.com/SAP/jenkins-library/pkg/config" "github.com/spf13/cobra" @@ -145,10 +145,11 @@ func processMetaFiles(metadataFiles []string, openFile func(s string) (io.ReadCl fmt.Printf("Step name: %v\n", stepData.Metadata.Name) - err = setDefaultParameters(&stepData) + osImport := false + osImport, err = setDefaultParameters(&stepData) checkError(err) - myStepInfo := getStepInfo(&stepData) + myStepInfo := getStepInfo(&stepData, osImport) step := stepTemplate(myStepInfo) err = writeFile(fmt.Sprintf("cmd/%v_generated.go", stepData.Metadata.Name), step, 0644) @@ -169,14 +170,16 @@ func fileWriter(filename string, data []byte, perm os.FileMode) error { return ioutil.WriteFile(filename, data, perm) } -func setDefaultParameters(stepData *config.StepData) error { +func setDefaultParameters(stepData *config.StepData) (bool, error) { //ToDo: custom function for default handling, support all relevant parameter types + osImportRequired := false for k, param := range stepData.Spec.Inputs.Parameters { if param.Default == nil { switch param.Type { case "string": param.Default = fmt.Sprintf("os.Getenv(\"PIPER_%v\")", param.Name) + osImportRequired = true case "bool": // ToDo: Check if default should be read from env param.Default = "false" @@ -184,7 +187,7 @@ func setDefaultParameters(stepData *config.StepData) error { // ToDo: Check if default should be read from env param.Default = "[]string{}" default: - return fmt.Errorf("Meta data type not set or not known: '%v'", param.Type) + return false, fmt.Errorf("Meta data type not set or not known: '%v'", param.Type) } } else { switch param.Type { @@ -199,16 +202,16 @@ func setDefaultParameters(stepData *config.StepData) error { case "[]string": param.Default = fmt.Sprintf("[]string{\"%v\"}", strings.Join(param.Default.([]string), "\", \"")) default: - return fmt.Errorf("Meta data type not set or not known: '%v'", param.Type) + return false, fmt.Errorf("Meta data type not set or not known: '%v'", param.Type) } } stepData.Spec.Inputs.Parameters[k] = param } - return nil + return osImportRequired, nil } -func getStepInfo(stepData *config.StepData) stepInfo { +func getStepInfo(stepData *config.StepData, osImport bool) stepInfo { return stepInfo{ StepName: stepData.Metadata.Name, CobraCmdFuncName: fmt.Sprintf("%vCommand", strings.Title(stepData.Metadata.Name)), @@ -217,6 +220,7 @@ func getStepInfo(stepData *config.StepData) stepInfo { Long: stepData.Metadata.LongDescription, Metadata: stepData.Spec.Inputs.Parameters, FlagsFunc: fmt.Sprintf("add%vFlags", strings.Title(stepData.Metadata.Name)), + OSImport: osImport, } } diff --git a/pkg/generator/step-metadata_test.go b/pkg/generator/step-metadata_test.go index 538c772f0..2af975ca7 100644 --- a/pkg/generator/step-metadata_test.go +++ b/pkg/generator/step-metadata_test.go @@ -112,10 +112,12 @@ func TestSetDefaultParameters(t *testing.T) { "[]string{}", } - err := setDefaultParameters(&stepData) + osImport, err := setDefaultParameters(&stepData) assert.NoError(t, err, "error occured but none expected") + assert.Equal(t, true, osImport, "import of os package required") + for k, v := range expected { assert.Equal(t, v, stepData.Spec.Inputs.Parameters[k].Default, fmt.Sprintf("default not correct for parameter %v", k)) } @@ -145,7 +147,7 @@ func TestSetDefaultParameters(t *testing.T) { } for k, v := range stepData { - err := setDefaultParameters(&v) + _, err := setDefaultParameters(&v) assert.Error(t, err, fmt.Sprintf("error expected but none occured for parameter %v", k)) } }) @@ -168,7 +170,7 @@ func TestGetStepInfo(t *testing.T) { }, } - myStepInfo := getStepInfo(&stepData) + myStepInfo := getStepInfo(&stepData, true) assert.Equal(t, "testStep", myStepInfo.StepName, "StepName incorrect") assert.Equal(t, "TestStepCommand", myStepInfo.CobraCmdFuncName, "CobraCmdFuncName incorrect") @@ -177,6 +179,7 @@ func TestGetStepInfo(t *testing.T) { assert.Equal(t, "Long Test description", myStepInfo.Long, "Long incorrect") assert.Equal(t, stepData.Spec.Inputs.Parameters, myStepInfo.Metadata, "Metadata incorrect") assert.Equal(t, "addTestStepFlags", myStepInfo.FlagsFunc, "FlagsFunc incorrect") + assert.Equal(t, "addTestStepFlags", myStepInfo.FlagsFunc, "FlagsFunc incorrect") } diff --git a/pkg/generator/testdata/TestProcessMetaFiles/step_code_generated.golden b/pkg/generator/testdata/TestProcessMetaFiles/step_code_generated.golden index 5aea334dd..ac667f25c 100644 --- a/pkg/generator/testdata/TestProcessMetaFiles/step_code_generated.golden +++ b/pkg/generator/testdata/TestProcessMetaFiles/step_code_generated.golden @@ -1,7 +1,7 @@ package cmd import ( - //"os" + "os" "github.com/SAP/jenkins-library/pkg/config" "github.com/spf13/cobra" diff --git a/pkg/github/github.go b/pkg/github/github.go new file mode 100644 index 000000000..5897a999c --- /dev/null +++ b/pkg/github/github.go @@ -0,0 +1,23 @@ +package github + +import ( + "context" + + "github.com/google/go-github/v28/github" + "golang.org/x/oauth2" +) + +//NewClient creates a new GitHub client using an OAuth token for authentication +func NewClient(token, apiURL, uploadURL string) (context.Context, *github.Client, error) { + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + + client, err := github.NewEnterpriseClient(apiURL, uploadURL, tc) + if err != nil { + return ctx, nil, err + } + return ctx, client, nil +} diff --git a/resources/metadata/githubrelease.yaml b/resources/metadata/githubrelease.yaml new file mode 100644 index 000000000..07b0ae1ce --- /dev/null +++ b/resources/metadata/githubrelease.yaml @@ -0,0 +1,132 @@ +metadata: + name: githubPublishRelease + description: Publish a release in GitHub + longDescription: | + This step creates a tag in your GitHub repository together with a release. + The release can be filled with text plus additional information like: + + * Closed pull request since last release + * Closed issues since last release + * Link to delta information showing all commits since last release + + The result looks like + + ![Example release](../images/githubRelease.png) +spec: + inputs: + secrets: + - name: githubTokenCredentialsId + description: Jenkins 'Secret text' credentials ID containing token to authenticate to GitHub. + type: jenkins + params: + - name: addClosedIssues + description: 'If set to `true`, closed issues and merged pull-requests since the last release will added below the `releaseBodyHeader`' + scope: + - PARAMETERS + - STAGES + - STEPS + type: bool + default: false + - name: addDeltaToLastRelease + description: 'If set to `true`, a link will be added to the relese information that brings up all commits since the last release.' + scope: + - PARAMETERS + - STAGES + - STEPS + type: bool + default: false + - name: assetPath + description: Path to a release asset which should be uploaded to the list of release assets. + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + - name: commitish + description: 'Target git commitish for the release' + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + default: "master" + - name: excludeLabels + description: 'Allows to exclude issues with dedicated list of labels.' + scope: + - PARAMETERS + - STAGES + - STEPS + type: '[]string' + - name: githubApiUrl + description: Set the GitHub API url. + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + type: string + default: https://api.github.com + mandatory: true + - name: githubOrg + description: 'Set the GitHub organization.' + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + mandatory: true + - name: githubRepo + description: 'Set the GitHub repository.' + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + mandatory: true + - name: githubServerUrl + description: 'GitHub server url for end-user access.' + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + default: https://github.com + mandatory: true + - name: githubToken + description: 'GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line' + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + type: string + mandatory: true + - name: labels + description: 'Labels to include in issue search.' + scope: + - PARAMETERS + - STAGES + - STEPS + type: '[]string' + - name: releaseBodyHeader + description: Content which will appear for the release. + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + - name: update + description: Specify if the release should be updated in case it already exists + scope: + - PARAMETERS + - STAGES + - STEPS + type: bool + - name: version + description: 'Define the version number which will be written as tag as well as release name.' + scope: + - PARAMETERS + - STAGES + - STEPS + type: string + mandatory: true From c241066e21dfbfd386dd71c0f947db124c70848d Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Mon, 4 Nov 2019 16:29:39 +0100 Subject: [PATCH 03/13] Add capability for hierarchical defaults in golang --- cmd/getConfig.go | 2 - pkg/config/stepmeta.go | 161 +++++++++++++++++++++++++----------- pkg/config/stepmeta_test.go | 59 +++++++++---- 3 files changed, 157 insertions(+), 65 deletions(-) diff --git a/cmd/getConfig.go b/cmd/getConfig.go index bbc4cc799..ec57dda1d 100644 --- a/cmd/getConfig.go +++ b/cmd/getConfig.go @@ -83,8 +83,6 @@ func generateConfig() error { return errors.Wrap(err, "getting step config failed") } - //ToDo: Check for mandatory parameters - myConfigJSON, _ := config.GetJSON(stepConfig.Config) fmt.Println(myConfigJSON) diff --git a/pkg/config/stepmeta.go b/pkg/config/stepmeta.go index 11d764eb0..778ed1169 100644 --- a/pkg/config/stepmeta.go +++ b/pkg/config/stepmeta.go @@ -48,6 +48,7 @@ type StepParameters struct { Mandatory bool `json:"mandatory,omitempty"` Default interface{} `json:"default,omitempty"` Aliases []Alias `json:"aliases,omitempty"` + Conditions []Condition `json:"conditions,omitempty"` } // Alias defines a step input parameter alias @@ -58,9 +59,10 @@ type Alias struct { // StepResources defines the resources to be provided by the step context, e.g. Jenkins pipeline type StepResources struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Type string `json:"type,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Type string `json:"type,omitempty"` + Conditions []Condition `json:"conditions,omitempty"` } // StepSecrets defines the secrets to be provided by the step context, e.g. Jenkins pipeline @@ -78,14 +80,15 @@ type StepSecrets struct { // Container defines an execution container type Container struct { //ToDo: check dockerOptions, dockerVolumeBind, containerPortMappings, sidecarOptions, sidecarVolumeBind - Command []string `json:"command"` - EnvVars []EnvVar `json:"env"` - Image string `json:"image"` - ImagePullPolicy string `json:"imagePullPolicy"` - Name string `json:"name"` - ReadyCommand string `json:"readyCommand"` - Shell string `json:"shell"` - WorkingDir string `json:"workingDir"` + Command []string `json:"command"` + EnvVars []EnvVar `json:"env"` + Image string `json:"image"` + ImagePullPolicy string `json:"imagePullPolicy"` + Name string `json:"name"` + ReadyCommand string `json:"readyCommand"` + Shell string `json:"shell"` + WorkingDir string `json:"workingDir"` + Conditions []Condition `json:"conditions,omitempty"` } // EnvVar defines an environment variable @@ -94,6 +97,18 @@ type EnvVar struct { Value string `json:"value"` } +// Condition defines an condition which decides when the parameter, resource or container is valid +type Condition struct { + ConditionRef string `json:"conditionRef"` + Params []Param `json:"params"` +} + +// Param defines the parameters serving as inputs to the condition +type Param struct { + Name string `json:"name"` + Value string `json:"value"` +} + // StepFilters defines the filter parameters for the different sections type StepFilters struct { All []string @@ -123,19 +138,25 @@ func (m *StepData) ReadPipelineStepData(metadata io.ReadCloser) error { func (m *StepData) GetParameterFilters() StepFilters { var filters StepFilters for _, param := range m.Spec.Inputs.Parameters { - filters.All = append(filters.All, param.Name) + parameterKeys := []string{param.Name} + for _, condition := range param.Conditions { + for _, dependentParam := range condition.Params { + parameterKeys = append(parameterKeys, dependentParam.Value) + } + } + filters.All = append(filters.All, parameterKeys...) for _, scope := range param.Scope { switch scope { case "GENERAL": - filters.General = append(filters.General, param.Name) + filters.General = append(filters.General, parameterKeys...) case "STEPS": - filters.Steps = append(filters.Steps, param.Name) + filters.Steps = append(filters.Steps, parameterKeys...) case "STAGES": - filters.Stages = append(filters.Stages, param.Name) + filters.Stages = append(filters.Stages, parameterKeys...) case "PARAMETERS": - filters.Parameters = append(filters.Parameters, param.Name) + filters.Parameters = append(filters.Parameters, parameterKeys...) case "ENV": - filters.Env = append(filters.Env, param.Name) + filters.Env = append(filters.Env, parameterKeys...) } } } @@ -156,7 +177,15 @@ func (m *StepData) GetContextParameterFilters() StepFilters { containerFilters := []string{} if len(m.Spec.Containers) > 0 { - containerFilters = append(containerFilters, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}...) + parameterKeys := []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"} + for _, container := range m.Spec.Containers { + for _, condition := range container.Conditions { + for _, dependentParam := range condition.Params { + parameterKeys = append(parameterKeys, dependentParam.Value) + } + } + } + containerFilters = append(containerFilters, parameterKeys...) } if len(m.Spec.Sidecars) > 0 { //ToDo: support fallback for "dockerName" configuration property -> via aliasing? @@ -175,59 +204,93 @@ func (m *StepData) GetContextParameterFilters() StepFilters { // It only supports scenarios with one container and optionally one sidecar func (m *StepData) GetContextDefaults(stepName string) (io.ReadCloser, error) { - p := map[string]interface{}{} - //ToDo error handling empty Containers/Sidecars //ToDo handle empty Command - + root := map[string]interface{}{} if len(m.Spec.Containers) > 0 { - if len(m.Spec.Containers[0].Command) > 0 { - p["containerCommand"] = m.Spec.Containers[0].Command[0] - } - p["containerName"] = m.Spec.Containers[0].Name - p["containerShell"] = m.Spec.Containers[0].Shell - p["dockerEnvVars"] = envVarsAsStringSlice(m.Spec.Containers[0].EnvVars) - p["dockerImage"] = m.Spec.Containers[0].Image - p["dockerName"] = m.Spec.Containers[0].Name - p["dockerPullImage"] = m.Spec.Containers[0].ImagePullPolicy != "Never" - p["dockerWorkspace"] = m.Spec.Containers[0].WorkingDir + for _, container := range m.Spec.Containers { + key := "" + if len(container.Conditions) > 0 { + key = container.Conditions[0].Params[0].Value + } + p := map[string]interface{}{} + if key != "" { + root[key] = p + } else { + p = root + } + if len(container.Command) > 0 { + p["containerCommand"] = container.Command[0] + } + p["containerName"] = container.Name + p["containerShell"] = container.Shell + p["dockerEnvVars"] = envVarsAsStringSlice(container.EnvVars) + p["dockerImage"] = container.Image + p["dockerName"] = container.Name + p["dockerPullImage"] = container.ImagePullPolicy != "Never" + p["dockerWorkspace"] = container.WorkingDir + + // Ready command not relevant for main runtime container so far + //p[] = container.ReadyCommand + } - // Ready command not relevant for main runtime container so far - //p[] = m.Spec.Containers[0].ReadyCommand } if len(m.Spec.Sidecars) > 0 { if len(m.Spec.Sidecars[0].Command) > 0 { - p["sidecarCommand"] = m.Spec.Sidecars[0].Command[0] + root["sidecarCommand"] = m.Spec.Sidecars[0].Command[0] } - p["sidecarEnvVars"] = envVarsAsStringSlice(m.Spec.Sidecars[0].EnvVars) - p["sidecarImage"] = m.Spec.Sidecars[0].Image - p["sidecarName"] = m.Spec.Sidecars[0].Name - p["sidecarPullImage"] = m.Spec.Sidecars[0].ImagePullPolicy != "Never" - p["sidecarReadyCommand"] = m.Spec.Sidecars[0].ReadyCommand - p["sidecarWorkspace"] = m.Spec.Sidecars[0].WorkingDir + root["sidecarEnvVars"] = envVarsAsStringSlice(m.Spec.Sidecars[0].EnvVars) + root["sidecarImage"] = m.Spec.Sidecars[0].Image + root["sidecarName"] = m.Spec.Sidecars[0].Name + root["sidecarPullImage"] = m.Spec.Sidecars[0].ImagePullPolicy != "Never" + root["sidecarReadyCommand"] = m.Spec.Sidecars[0].ReadyCommand + root["sidecarWorkspace"] = m.Spec.Sidecars[0].WorkingDir } // not filled for now since this is not relevant in Kubernetes case - //p["dockerOptions"] = m.Spec.Containers[0]. - //p["dockerVolumeBind"] = m.Spec.Containers[0]. - //p["containerPortMappings"] = m.Spec.Sidecars[0]. - //p["sidecarOptions"] = m.Spec.Sidecars[0]. - //p["sidecarVolumeBind"] = m.Spec.Sidecars[0]. + //p["dockerOptions"] = container. + //p["dockerVolumeBind"] = container. + //root["containerPortMappings"] = m.Spec.Sidecars[0]. + //root["sidecarOptions"] = m.Spec.Sidecars[0]. + //root["sidecarVolumeBind"] = m.Spec.Sidecars[0]. if len(m.Spec.Inputs.Resources) > 0 { - var resources []string + keys := []string{} + resources := map[string][]string{} for _, resource := range m.Spec.Inputs.Resources { if resource.Type == "stash" { - resources = append(resources, resource.Name) + key := "" + if len(resource.Conditions) > 0 { + key = resource.Conditions[0].Params[0].Value + } + if resources[key] == nil { + keys = append(keys, key) + resources[key] = []string{} + } + resources[key] = append(resources[key], resource.Name) + } + } + + for _, key := range keys { + if key == "" { + root["stashContent"] = resources[""] + } else { + if root[key] == nil { + root[key] = map[string]interface{}{ + "stashContent": resources[key], + } + } else { + p := root[key].(map[string]interface{}) + p["stashContent"] = resources[key] + } } } - p["stashContent"] = resources } c := Config{ Steps: map[string]map[string]interface{}{ - stepName: p, + stepName: root, }, } diff --git a/pkg/config/stepmeta_test.go b/pkg/config/stepmeta_test.go index f5ba27fa2..7917ade3b 100644 --- a/pkg/config/stepmeta_test.go +++ b/pkg/config/stepmeta_test.go @@ -68,6 +68,7 @@ func TestGetParameterFilters(t *testing.T) { {Name: "paramFour", Scope: []string{"PARAMETERS", "ENV"}}, {Name: "paramFive", Scope: []string{"ENV"}}, {Name: "paramSix"}, + {Name: "paramSeven", Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"}, Conditions: []Condition{{Params: []Param{{Name: "buildTool", Value: "mta"}}}}}, }, }, }, @@ -113,17 +114,17 @@ func TestGetParameterFilters(t *testing.T) { }{ { Metadata: metadata1, - ExpectedGeneral: []string{"paramOne"}, - ExpectedSteps: []string{"paramOne", "paramTwo"}, - ExpectedStages: []string{"paramOne", "paramTwo", "paramThree"}, - ExpectedParameters: []string{"paramOne", "paramTwo", "paramThree", "paramFour"}, - ExpectedEnv: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive"}, - ExpectedAll: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"}, + ExpectedGeneral: []string{"paramOne", "paramSeven", "mta"}, + ExpectedSteps: []string{"paramOne", "paramTwo", "paramSeven", "mta"}, + ExpectedStages: []string{"paramOne", "paramTwo", "paramThree", "paramSeven", "mta"}, + ExpectedParameters: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramSeven", "mta"}, + ExpectedEnv: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSeven", "mta"}, + ExpectedAll: []string{"paramOne", "paramTwo", "paramThree", "paramFour", "paramFive", "paramSix", "paramSeven", "mta"}, NotExpectedGeneral: []string{"paramTwo", "paramThree", "paramFour", "paramFive", "paramSix"}, NotExpectedSteps: []string{"paramThree", "paramFour", "paramFive", "paramSix"}, NotExpectedStages: []string{"paramFour", "paramFive", "paramSix"}, NotExpectedParameters: []string{"paramFive", "paramSix"}, - NotExpectedEnv: []string{"paramSix"}, + NotExpectedEnv: []string{"paramSix", "mta"}, NotExpectedAll: []string{}, }, { @@ -234,6 +235,11 @@ func TestGetContextParameterFilters(t *testing.T) { Spec: StepSpec{ Containers: []Container{ {Name: "testcontainer"}, + {Conditions: []Condition{ + {Params: []Param{ + {Name: "scanType", Value: "pip"}, + }}, + }}, }, }, } @@ -258,12 +264,12 @@ func TestGetContextParameterFilters(t *testing.T) { t.Run("Containers", func(t *testing.T) { filters := metadata2.GetContextParameterFilters() - assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.All, "incorrect filter All") - assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.General, "incorrect filter General") - assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Steps, "incorrect filter Steps") - assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Stages, "incorrect filter Stages") - assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Parameters, "incorrect filter Parameters") - assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace"}, filters.Env, "incorrect filter Env") + assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.All, "incorrect filter All") + assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.General, "incorrect filter General") + assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Steps, "incorrect filter Steps") + assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Stages, "incorrect filter Stages") + assert.Equal(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Parameters, "incorrect filter Parameters") + assert.NotEqual(t, []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "pip"}, filters.Env, "incorrect filter Env") }) t.Run("Sidecars", func(t *testing.T) { @@ -287,15 +293,38 @@ func TestGetContextDefaults(t *testing.T) { { Name: "buildDescriptor", Type: "stash", + Conditions: []Condition{ + {Params: []Param{ + {Name: "scanType", Value: "abc"}, + }}, + }, }, { Name: "source", Type: "stash", + Conditions: []Condition{ + {Params: []Param{ + {Name: "scanType", Value: "abc"}, + }}, + }, }, { Name: "test", Type: "nonce", }, + { + Name: "test2", + Type: "stash", + Conditions: []Condition{ + {Params: []Param{ + {Name: "scanType", Value: "def"}, + }}, + }, + }, + { + Name: "test3", + Type: "stash", + }, }, }, Containers: []Container{ @@ -339,7 +368,9 @@ func TestGetContextDefaults(t *testing.T) { var d PipelineDefaults d.ReadPipelineDefaults([]io.ReadCloser{cd}) - assert.Equal(t, []interface{}{"buildDescriptor", "source"}, d.Defaults[0].Steps["testStep"]["stashContent"], "stashContent default not available") + assert.Equal(t, []interface{}{"buildDescriptor", "source"}, d.Defaults[0].Steps["testStep"]["abc"].(map[string]interface{})["stashContent"], "stashContent default not available") + assert.Equal(t, []interface{}{"test2"}, d.Defaults[0].Steps["testStep"]["def"].(map[string]interface{})["stashContent"], "stashContent default not available") + assert.Equal(t, []interface{}{"test3"}, d.Defaults[0].Steps["testStep"]["stashContent"], "stashContent default not available") assert.Equal(t, "test/command", d.Defaults[0].Steps["testStep"]["containerCommand"], "containerCommand default not available") assert.Equal(t, "testcontainer", d.Defaults[0].Steps["testStep"]["containerName"], "containerName default not available") assert.Equal(t, "/bin/bash", d.Defaults[0].Steps["testStep"]["containerShell"], "containerShell default not available") From 50153f42c73eea70643db9649fb0e9799cd10b22 Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Tue, 5 Nov 2019 14:37:44 +0100 Subject: [PATCH 04/13] Add tests --- cmd/githubPublishRelease.go | 102 +++++- cmd/githubPublishRelease_generated.go | 18 +- cmd/githubPublishRelease_test.go | 346 +++++++++++++++++- .../Success_-_update_asset_test.txt | 1 + .../Success_-_existing_asset_test.txt | 1 + .../Success_-_no_asset_test.txt | 1 + cmd/version_generated.go | 2 - cmd/version_test.go | 21 +- resources/metadata/githubrelease.yaml | 14 +- 9 files changed, 472 insertions(+), 34 deletions(-) create mode 100644 cmd/testdata/TestRunGithubPublishRelease/Success_-_update_asset_test.txt create mode 100644 cmd/testdata/TestUploadReleaseAsset/Success_-_existing_asset_test.txt create mode 100644 cmd/testdata/TestUploadReleaseAsset/Success_-_no_asset_test.txt diff --git a/cmd/githubPublishRelease.go b/cmd/githubPublishRelease.go index aa6044303..701bdeb54 100644 --- a/cmd/githubPublishRelease.go +++ b/cmd/githubPublishRelease.go @@ -3,8 +3,12 @@ package cmd import ( "context" "fmt" + "mime" + "os" + "path/filepath" "strings" + "github.com/SAP/jenkins-library/pkg/log" "github.com/google/go-github/v28/github" "github.com/pkg/errors" @@ -12,8 +16,11 @@ import ( ) type githubRepoClient interface { - GetLatestRelease(ctx context.Context, owner string, repo string) (*github.RepositoryRelease, *github.Response, error) CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) + DeleteReleaseAsset(ctx context.Context, owner string, repo string, id int64) (*github.Response, error) + GetLatestRelease(ctx context.Context, owner string, repo string) (*github.RepositoryRelease, *github.Response, error) + ListReleaseAssets(ctx context.Context, owner string, repo string, id int64, opt *github.ListOptions) ([]*github.ReleaseAsset, *github.Response, error) + UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) } type githubIssueClient interface { @@ -21,14 +28,14 @@ type githubIssueClient interface { } func githubPublishRelease(myGithubPublishReleaseOptions githubPublishReleaseOptions) error { - ctx, client, err := piperGithub.NewClient(myGithubPublishReleaseOptions.GithubToken, myGithubPublishReleaseOptions.GithubAPIURL, myGithubPublishReleaseOptions.GithubAPIURL) + ctx, client, err := piperGithub.NewClient(myGithubPublishReleaseOptions.GithubToken, myGithubPublishReleaseOptions.GithubAPIURL, myGithubPublishReleaseOptions.GithubUploadURL) if err != nil { - return err + log.Entry().WithError(err).Fatal("Failed to get GitHub client.") } err = runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, client.Repositories, client.Issues) if err != nil { - return err + log.Entry().WithError(err).Fatal("Failed to publish GitHub release.") } return nil @@ -37,21 +44,28 @@ func githubPublishRelease(myGithubPublishReleaseOptions githubPublishReleaseOpti func runGithubPublishRelease(ctx context.Context, myGithubPublishReleaseOptions *githubPublishReleaseOptions, ghRepoClient githubRepoClient, ghIssueClient githubIssueClient) error { var publishedAt github.Timestamp + lastRelease, resp, err := ghRepoClient.GetLatestRelease(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo) if err != nil { if resp.StatusCode == 404 { - //first release + //no previous release found -> first release myGithubPublishReleaseOptions.AddDeltaToLastRelease = false - publishedAt = lastRelease.GetPublishedAt() + log.Entry().Debug("This is the first release.") } else { - return errors.Wrap(err, "Error occured when retrieving latest GitHub releass") + return errors.Wrap(err, "Error occured when retrieving latest GitHub release.") } } + publishedAt = lastRelease.GetPublishedAt() + log.Entry().Debugf("Previous GitHub release published: '%v'", publishedAt) + + if myGithubPublishReleaseOptions.UpdateAsset { + return uploadReleaseAsset(ctx, lastRelease.GetID(), myGithubPublishReleaseOptions, ghRepoClient) + } releaseBody := "" if len(myGithubPublishReleaseOptions.ReleaseBodyHeader) > 0 { - releaseBody += myGithubPublishReleaseOptions.ReleaseBodyHeader + "
" + releaseBody += myGithubPublishReleaseOptions.ReleaseBodyHeader + "\n" } if myGithubPublishReleaseOptions.AddClosedIssues { @@ -69,20 +83,22 @@ func runGithubPublishRelease(ctx context.Context, myGithubPublishReleaseOptions Body: &releaseBody, } - //create release createdRelease, _, err := ghRepoClient.CreateRelease(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, &release) if err != nil { - return errors.Wrapf(err, "creation of release '%v' failed", release.TagName) + return errors.Wrapf(err, "Creation of release '%v' failed", *release.TagName) } + log.Entry().Infof("Release %v created on %v/%v", *createdRelease.TagName, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo) - // todo switch to logging - fmt.Printf("Release %v created on %v/%v", *createdRelease.TagName, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo) + if len(myGithubPublishReleaseOptions.AssetPath) > 0 { + return uploadReleaseAsset(ctx, createdRelease.GetID(), myGithubPublishReleaseOptions, ghRepoClient) + } return nil } func getClosedIssuesText(ctx context.Context, publishedAt github.Timestamp, myGithubPublishReleaseOptions *githubPublishReleaseOptions, ghIssueClient githubIssueClient) string { closedIssuesText := "" + options := github.IssueListByRepoOptions{ State: "closed", Direction: "asc", @@ -93,17 +109,19 @@ func getClosedIssuesText(ctx context.Context, publishedAt github.Timestamp, myGi } ghIssues, _, err := ghIssueClient.ListByRepo(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, &options) if err != nil { - //log error + log.Entry().WithError(err).Error("Failed to get GitHub issues.") } - prTexts := []string{"
**List of closed pull-requests since last release**"} - issueTexts := []string{"
**List of closed issues since last release**"} + prTexts := []string{"\n**List of closed pull-requests since last release**"} + issueTexts := []string{"\n**List of closed issues since last release**"} for _, issue := range ghIssues { if issue.IsPullRequest() && !isExcluded(issue, myGithubPublishReleaseOptions.ExcludeLabels) { prTexts = append(prTexts, fmt.Sprintf("[#%v](%v): %v", issue.GetNumber(), issue.GetHTMLURL(), issue.GetTitle())) + log.Entry().Debugf("Added PR #%v to release", issue.GetNumber()) } else if !issue.IsPullRequest() && !isExcluded(issue, myGithubPublishReleaseOptions.ExcludeLabels) { issueTexts = append(issueTexts, fmt.Sprintf("[#%v](%v): %v", issue.GetNumber(), issue.GetHTMLURL(), issue.GetTitle())) + log.Entry().Debugf("Added Issue #%v to release", issue.GetNumber()) } } @@ -121,9 +139,9 @@ func getReleaseDeltaText(myGithubPublishReleaseOptions *githubPublishReleaseOpti releaseDeltaText := "" //add delta link to previous release - releaseDeltaText += "
**Changes**
" + releaseDeltaText += "\n**Changes**\n" releaseDeltaText += fmt.Sprintf( - "[%v...%v](%v/%v/%v/compare/%v...%v)
", + "[%v...%v](%v/%v/%v/compare/%v...%v)\n", lastRelease.GetTagName(), myGithubPublishReleaseOptions.Version, myGithubPublishReleaseOptions.GithubServerURL, @@ -135,6 +153,56 @@ func getReleaseDeltaText(myGithubPublishReleaseOptions *githubPublishReleaseOpti return releaseDeltaText } +func uploadReleaseAsset(ctx context.Context, releaseID int64, myGithubPublishReleaseOptions *githubPublishReleaseOptions, ghRepoClient githubRepoClient) error { + + assets, _, err := ghRepoClient.ListReleaseAssets(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, releaseID, &github.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Failed to get list of release assets.") + } + var assetID int64 + for _, a := range assets { + if a.GetName() == filepath.Base(myGithubPublishReleaseOptions.AssetPath) { + assetID = a.GetID() + break + } + } + if assetID != 0 { + //asset needs to be deleted first since API does not allow for replacement + _, err := ghRepoClient.DeleteReleaseAsset(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, assetID) + if err != nil { + return errors.Wrap(err, "Failed to delete release asset.") + } + } + + mediaType := mime.TypeByExtension(filepath.Ext(myGithubPublishReleaseOptions.AssetPath)) + if mediaType == "" { + mediaType = "application/octet-stream" + } + log.Entry().Debugf("Using mediaType '%v'", mediaType) + + name := filepath.Base(myGithubPublishReleaseOptions.AssetPath) + log.Entry().Debugf("Using file name '%v'", name) + + opts := github.UploadOptions{ + Name: name, + MediaType: mediaType, + } + file, err := os.Open(myGithubPublishReleaseOptions.AssetPath) + defer file.Close() + if err != nil { + return errors.Wrapf(err, "Failed to load release asset '%v'", myGithubPublishReleaseOptions.AssetPath) + } + + log.Entry().Info("Starting to upload release asset.") + asset, _, err := ghRepoClient.UploadReleaseAsset(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, releaseID, &opts, file) + if err != nil { + return errors.Wrap(err, "Failed to upload release asset.") + } + log.Entry().Infof("Done uploading asset '%v'.", asset.GetURL()) + + return nil +} + func isExcluded(issue *github.Issue, excludeLabels []string) bool { //issue.Labels[0].GetName() for _, ex := range excludeLabels { diff --git a/cmd/githubPublishRelease_generated.go b/cmd/githubPublishRelease_generated.go index 10cc6e2aa..3fc00a1b3 100644 --- a/cmd/githubPublishRelease_generated.go +++ b/cmd/githubPublishRelease_generated.go @@ -4,6 +4,7 @@ import ( "os" "github.com/SAP/jenkins-library/pkg/config" + "github.com/SAP/jenkins-library/pkg/log" "github.com/spf13/cobra" ) @@ -18,9 +19,10 @@ type githubPublishReleaseOptions struct { GithubRepo string `json:"githubRepo,omitempty"` GithubServerURL string `json:"githubServerUrl,omitempty"` GithubToken string `json:"githubToken,omitempty"` + GithubUploadURL string `json:"githubUploadUrl,omitempty"` Labels []string `json:"labels,omitempty"` ReleaseBodyHeader string `json:"releaseBodyHeader,omitempty"` - Update bool `json:"update,omitempty"` + UpdateAsset bool `json:"updateAsset,omitempty"` Version string `json:"version,omitempty"` } @@ -44,6 +46,8 @@ The result looks like ![Example release](../images/githubRelease.png)`, PreRunE: func(cmd *cobra.Command, args []string) error { + log.SetStepName("githubPublishRelease") + log.SetVerbose(generalConfig.verbose) return PrepareConfig(cmd, &metadata, "githubPublishRelease", &myGithubPublishReleaseOptions, openPiperFile) }, RunE: func(cmd *cobra.Command, args []string) error { @@ -66,9 +70,10 @@ func addGithubPublishReleaseFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubRepo, "githubRepo", os.Getenv("PIPER_githubRepo"), "Set the GitHub repository.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubServerURL, "githubServerUrl", "https://github.com", "GitHub server url for end-user access.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubUploadURL, "githubUploadUrl", "https://uploads.github.com", "Set the GitHub API url.") cmd.Flags().StringSliceVar(&myGithubPublishReleaseOptions.Labels, "labels", []string{}, "Labels to include in issue search.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.ReleaseBodyHeader, "releaseBodyHeader", os.Getenv("PIPER_releaseBodyHeader"), "Content which will appear for the release.") - cmd.Flags().BoolVar(&myGithubPublishReleaseOptions.Update, "update", false, "Specify if the release should be updated in case it already exists") + cmd.Flags().BoolVar(&myGithubPublishReleaseOptions.UpdateAsset, "updateAsset", false, "Specify if a release asset should be updated only.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Version, "version", os.Getenv("PIPER_version"), "Define the version number which will be written as tag as well as release name.") cmd.MarkFlagRequired("githubApiUrl") @@ -76,6 +81,7 @@ func addGithubPublishReleaseFlags(cmd *cobra.Command) { cmd.MarkFlagRequired("githubRepo") cmd.MarkFlagRequired("githubServerUrl") cmd.MarkFlagRequired("githubToken") + cmd.MarkFlagRequired("githubUploadUrl") cmd.MarkFlagRequired("version") } @@ -145,6 +151,12 @@ func githubPublishReleaseMetadata() config.StepData { Type: "string", Mandatory: true, }, + { + Name: "githubUploadUrl", + Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: true, + }, { Name: "labels", Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, @@ -158,7 +170,7 @@ func githubPublishReleaseMetadata() config.StepData { Mandatory: false, }, { - Name: "update", + Name: "updateAsset", Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "bool", Mandatory: false, diff --git a/cmd/githubPublishRelease_test.go b/cmd/githubPublishRelease_test.go index 33ff34fa3..88ff54ec0 100644 --- a/cmd/githubPublishRelease_test.go +++ b/cmd/githubPublishRelease_test.go @@ -1,15 +1,359 @@ package cmd import ( + "context" "fmt" + "net/http" + "os" + "path/filepath" "testing" + "time" "github.com/google/go-github/v28/github" "github.com/stretchr/testify/assert" ) -func TestRunGithubPublishRelease(t *testing.T) { +type ghRCMock struct { + createErr error + latestRelease *github.RepositoryRelease + release *github.RepositoryRelease + delErr error + delID int64 + delOwner string + delRepo string + listErr error + listID int64 + listOwner string + listReleaseAssets []*github.ReleaseAsset + listRepo string + listOpts *github.ListOptions + latestStatusCode int + latestErr error + uploadID int64 + uploadOpts *github.UploadOptions + uploadOwner string + uploadRepo string +} +func (g *ghRCMock) CreateRelease(ctx context.Context, owner string, repo string, release *github.RepositoryRelease) (*github.RepositoryRelease, *github.Response, error) { + g.release = release + return release, nil, g.createErr +} + +func (g *ghRCMock) DeleteReleaseAsset(ctx context.Context, owner string, repo string, id int64) (*github.Response, error) { + g.delOwner = owner + g.delRepo = repo + g.delID = id + return nil, g.delErr +} + +func (g *ghRCMock) GetLatestRelease(ctx context.Context, owner string, repo string) (*github.RepositoryRelease, *github.Response, error) { + hc := http.Response{StatusCode: 200} + if g.latestStatusCode != 0 { + hc.StatusCode = g.latestStatusCode + } + ghResp := github.Response{Response: &hc} + return g.latestRelease, &ghResp, g.latestErr +} + +func (g *ghRCMock) ListReleaseAssets(ctx context.Context, owner string, repo string, id int64, opt *github.ListOptions) ([]*github.ReleaseAsset, *github.Response, error) { + g.listID = id + g.listOwner = owner + g.listRepo = repo + g.listOpts = opt + return g.listReleaseAssets, nil, g.listErr +} + +func (g *ghRCMock) UploadReleaseAsset(ctx context.Context, owner string, repo string, id int64, opt *github.UploadOptions, file *os.File) (*github.ReleaseAsset, *github.Response, error) { + g.uploadID = id + g.uploadOwner = owner + g.uploadRepo = repo + g.uploadOpts = opt + return nil, nil, nil +} + +type ghICMock struct { + issues []*github.Issue + lastPublished time.Time + owner string + repo string + options *github.IssueListByRepoOptions +} + +func (g *ghICMock) ListByRepo(ctx context.Context, owner string, repo string, opt *github.IssueListByRepoOptions) ([]*github.Issue, *github.Response, error) { + g.owner = owner + g.repo = repo + g.options = opt + g.lastPublished = opt.Since + return g.issues, nil, nil +} + +func TestRunGithubPublishRelease(t *testing.T) { + ctx := context.Background() + + t.Run("Success - first release & no body", func(t *testing.T) { + ghIssueClient := ghICMock{} + ghRepoClient := ghRCMock{ + latestStatusCode: 404, + latestErr: fmt.Errorf("not found"), + } + + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + AddDeltaToLastRelease: true, + Commitish: "master", + GithubOrg: "TEST", + GithubRepo: "test", + GithubServerURL: "https://github.com", + ReleaseBodyHeader: "Header", + Version: "1.0", + } + err := runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, &ghRepoClient, &ghIssueClient) + assert.NoError(t, err, "Error occured but none expected.") + + assert.Equal(t, "Header\n", ghRepoClient.release.GetBody()) + }) + + t.Run("Success - subsequent releases & with body", func(t *testing.T) { + lastTag := "1.0" + lastPublishedAt := github.Timestamp{Time: time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC)} + ghRepoClient := ghRCMock{ + createErr: nil, + latestRelease: &github.RepositoryRelease{ + TagName: &lastTag, + PublishedAt: &lastPublishedAt, + }, + } + prHTMLURL := "https://github.com/TEST/test/pull/1" + prTitle := "Pull" + prNo := 1 + + issHTMLURL := "https://github.com/TEST/test/issues/2" + issTitle := "Issue" + issNo := 2 + + ghIssueClient := ghICMock{ + issues: []*github.Issue{ + {Number: &prNo, Title: &prTitle, HTMLURL: &prHTMLURL, PullRequestLinks: &github.PullRequestLinks{URL: &prHTMLURL}}, + {Number: &issNo, Title: &issTitle, HTMLURL: &issHTMLURL}, + }, + } + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + AddClosedIssues: true, + AddDeltaToLastRelease: true, + Commitish: "master", + GithubOrg: "TEST", + GithubRepo: "test", + GithubServerURL: "https://github.com", + ReleaseBodyHeader: "Header", + Version: "1.1", + } + err := runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, &ghRepoClient, &ghIssueClient) + + assert.NoError(t, err, "Error occured but none expected.") + + assert.Equal(t, "Header\n\n**List of closed pull-requests since last release**\n[#1](https://github.com/TEST/test/pull/1): Pull\n\n**List of closed issues since last release**\n[#2](https://github.com/TEST/test/issues/2): Issue\n\n**Changes**\n[1.0...1.1](https://github.com/TEST/test/compare/1.0...1.1)\n", ghRepoClient.release.GetBody()) + assert.Equal(t, "1.1", ghRepoClient.release.GetName()) + assert.Equal(t, "1.1", ghRepoClient.release.GetTagName()) + assert.Equal(t, "master", ghRepoClient.release.GetTargetCommitish()) + + assert.Equal(t, lastPublishedAt.Time, ghIssueClient.lastPublished) + }) + + t.Run("Success - update asset", func(t *testing.T) { + var releaseID int64 = 1 + ghIssueClient := ghICMock{} + ghRepoClient := ghRCMock{ + latestRelease: &github.RepositoryRelease{ + ID: &releaseID, + }, + } + + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + UpdateAsset: true, + AssetPath: filepath.Join("testdata", t.Name()+"_test.txt"), + } + + err := runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, &ghRepoClient, &ghIssueClient) + + assert.NoError(t, err, "Error occured but none expected.") + + assert.Nil(t, ghRepoClient.release) + + assert.Equal(t, releaseID, ghRepoClient.listID) + assert.Equal(t, releaseID, ghRepoClient.uploadID) + }) + + t.Run("Error - get release", func(t *testing.T) { + ghIssueClient := ghICMock{} + ghRepoClient := ghRCMock{ + latestErr: fmt.Errorf("Latest release error"), + } + myGithubPublishReleaseOptions := githubPublishReleaseOptions{} + err := runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, &ghRepoClient, &ghIssueClient) + + assert.Equal(t, "Error occured when retrieving latest GitHub release.: Latest release error", fmt.Sprint(err)) + }) + + t.Run("Error - create release", func(t *testing.T) { + ghIssueClient := ghICMock{} + ghRepoClient := ghRCMock{ + createErr: fmt.Errorf("Create release error"), + } + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + Version: "1.0", + } + err := runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, &ghRepoClient, &ghIssueClient) + + assert.Equal(t, "Creation of release '1.0' failed: Create release error", fmt.Sprint(err)) + }) +} + +func TestGetClosedIssuesText(t *testing.T) { + ctx := context.Background() + publishedAt := github.Timestamp{Time: time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC)} + + t.Run("No issues", func(t *testing.T) { + ghIssueClient := ghICMock{} + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + Version: "1.0", + } + + res := getClosedIssuesText(ctx, publishedAt, &myGithubPublishReleaseOptions, &ghIssueClient) + + assert.Equal(t, "", res) + }) + + t.Run("All issues", func(t *testing.T) { + ctx := context.Background() + publishedAt := github.Timestamp{Time: time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC)} + + prHTMLURL := []string{"https://github.com/TEST/test/pull/1", "https://github.com/TEST/test/pull/2"} + prTitle := []string{"Pull1", "Pull2"} + prNo := []int{1, 2} + + issHTMLURL := []string{"https://github.com/TEST/test/issues/3", "https://github.com/TEST/test/issues/4"} + issTitle := []string{"Issue3", "Issue4"} + issNo := []int{3, 4} + + ghIssueClient := ghICMock{ + issues: []*github.Issue{ + {Number: &prNo[0], Title: &prTitle[0], HTMLURL: &prHTMLURL[0], PullRequestLinks: &github.PullRequestLinks{URL: &prHTMLURL[0]}}, + {Number: &prNo[1], Title: &prTitle[1], HTMLURL: &prHTMLURL[1], PullRequestLinks: &github.PullRequestLinks{URL: &prHTMLURL[1]}}, + {Number: &issNo[0], Title: &issTitle[0], HTMLURL: &issHTMLURL[0]}, + {Number: &issNo[1], Title: &issTitle[1], HTMLURL: &issHTMLURL[1]}, + }, + } + + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + GithubOrg: "TEST", + GithubRepo: "test", + } + + res := getClosedIssuesText(ctx, publishedAt, &myGithubPublishReleaseOptions, &ghIssueClient) + + assert.Equal(t, "\n**List of closed pull-requests since last release**\n[#1](https://github.com/TEST/test/pull/1): Pull1\n[#2](https://github.com/TEST/test/pull/2): Pull2\n\n**List of closed issues since last release**\n[#3](https://github.com/TEST/test/issues/3): Issue3\n[#4](https://github.com/TEST/test/issues/4): Issue4\n", res) + assert.Equal(t, "TEST", ghIssueClient.owner, "Owner not properly passed") + assert.Equal(t, "test", ghIssueClient.repo, "Repo not properly passed") + assert.Equal(t, "closed", ghIssueClient.options.State, "Issue state not properly passed") + assert.Equal(t, "asc", ghIssueClient.options.Direction, "Sort direction not properly passed") + assert.Equal(t, publishedAt.Time, ghIssueClient.options.Since, "PublishedAt not properly passed") + }) + +} + +func TestGetReleaseDeltaText(t *testing.T) { + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + GithubOrg: "TEST", + GithubRepo: "test", + GithubServerURL: "https://github.com", + Version: "1.1", + } + lastTag := "1.0" + lastRelease := github.RepositoryRelease{ + TagName: &lastTag, + } + + res := getReleaseDeltaText(&myGithubPublishReleaseOptions, &lastRelease) + + assert.Equal(t, "\n**Changes**\n[1.0...1.1](https://github.com/TEST/test/compare/1.0...1.1)\n", res) +} + +func TestUploadReleaseAsset(t *testing.T) { + ctx := context.Background() + + t.Run("Success - existing asset", func(t *testing.T) { + var releaseID int64 = 1 + assetName := "Success_-_existing_asset_test.txt" + var assetID int64 = 11 + ghRepoClient := ghRCMock{ + latestRelease: &github.RepositoryRelease{ + ID: &releaseID, + }, + listReleaseAssets: []*github.ReleaseAsset{ + {Name: &assetName, ID: &assetID}, + }, + } + + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + GithubOrg: "TEST", + GithubRepo: "test", + AssetPath: filepath.Join("testdata", t.Name()+"_test.txt"), + } + + err := uploadReleaseAsset(ctx, releaseID, &myGithubPublishReleaseOptions, &ghRepoClient) + + assert.NoError(t, err, "Error occured but none expected.") + + assert.Equal(t, "TEST", ghRepoClient.listOwner, "Owner not properly passed - list") + assert.Equal(t, "test", ghRepoClient.listRepo, "Repo not properly passed - list") + assert.Equal(t, releaseID, ghRepoClient.listID, "Relase ID not properly passed - list") + + assert.Equal(t, "TEST", ghRepoClient.delOwner, "Owner not properly passed - del") + assert.Equal(t, "test", ghRepoClient.delRepo, "Repo not properly passed - del") + assert.Equal(t, assetID, ghRepoClient.delID, "Relase ID not properly passed - del") + + assert.Equal(t, "TEST", ghRepoClient.uploadOwner, "Owner not properly passed - upload") + assert.Equal(t, "test", ghRepoClient.uploadRepo, "Repo not properly passed - upload") + assert.Equal(t, releaseID, ghRepoClient.uploadID, "Relase ID not properly passed - upload") + assert.Equal(t, "text/plain; charset=utf-8", ghRepoClient.uploadOpts.MediaType, "Wrong MediaType passed - upload") + }) + + t.Run("Success - no asset", func(t *testing.T) { + var releaseID int64 = 1 + assetName := "notFound" + var assetID int64 = 11 + ghRepoClient := ghRCMock{ + latestRelease: &github.RepositoryRelease{ + ID: &releaseID, + }, + listReleaseAssets: []*github.ReleaseAsset{ + {Name: &assetName, ID: &assetID}, + }, + } + + myGithubPublishReleaseOptions := githubPublishReleaseOptions{ + GithubOrg: "TEST", + GithubRepo: "test", + AssetPath: filepath.Join("testdata", t.Name()+"_test.txt"), + } + + err := uploadReleaseAsset(ctx, releaseID, &myGithubPublishReleaseOptions, &ghRepoClient) + + assert.NoError(t, err, "Error occured but none expected.") + + assert.Equal(t, int64(0), ghRepoClient.delID, "Relase ID should not be populated") + }) + + t.Run("Error - List Assets", func(t *testing.T) { + var releaseID int64 = 1 + ghRepoClient := ghRCMock{ + listErr: fmt.Errorf("List Asset Error"), + } + myGithubPublishReleaseOptions := githubPublishReleaseOptions{} + + err := uploadReleaseAsset(ctx, releaseID, &myGithubPublishReleaseOptions, &ghRepoClient) + assert.Equal(t, "Failed to get list of release assets.: List Asset Error", fmt.Sprint(err), "Wrong error received") + }) } func TestIsExcluded(t *testing.T) { diff --git a/cmd/testdata/TestRunGithubPublishRelease/Success_-_update_asset_test.txt b/cmd/testdata/TestRunGithubPublishRelease/Success_-_update_asset_test.txt new file mode 100644 index 000000000..3b1246497 --- /dev/null +++ b/cmd/testdata/TestRunGithubPublishRelease/Success_-_update_asset_test.txt @@ -0,0 +1 @@ +TEST \ No newline at end of file diff --git a/cmd/testdata/TestUploadReleaseAsset/Success_-_existing_asset_test.txt b/cmd/testdata/TestUploadReleaseAsset/Success_-_existing_asset_test.txt new file mode 100644 index 000000000..3b1246497 --- /dev/null +++ b/cmd/testdata/TestUploadReleaseAsset/Success_-_existing_asset_test.txt @@ -0,0 +1 @@ +TEST \ No newline at end of file diff --git a/cmd/testdata/TestUploadReleaseAsset/Success_-_no_asset_test.txt b/cmd/testdata/TestUploadReleaseAsset/Success_-_no_asset_test.txt new file mode 100644 index 000000000..3b1246497 --- /dev/null +++ b/cmd/testdata/TestUploadReleaseAsset/Success_-_no_asset_test.txt @@ -0,0 +1 @@ +TEST \ No newline at end of file diff --git a/cmd/version_generated.go b/cmd/version_generated.go index 2b4802299..d0d8c2152 100644 --- a/cmd/version_generated.go +++ b/cmd/version_generated.go @@ -1,8 +1,6 @@ package cmd import ( - //"os" - "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/log" "github.com/spf13/cobra" diff --git a/cmd/version_test.go b/cmd/version_test.go index 5135bb4b5..a3daffb87 100644 --- a/cmd/version_test.go +++ b/cmd/version_test.go @@ -1,11 +1,11 @@ package cmd import ( - "testing" - "os" "bytes" - "io" "github.com/stretchr/testify/assert" + "io" + "os" + "testing" ) func TestVersion(t *testing.T) { @@ -17,7 +17,6 @@ func TestVersion(t *testing.T) { assert.Contains(t, result, "tag: \"\"") }) - t.Run("versionAndTagSet", func(t *testing.T) { result := runVersionCommand(t, "16bafe", "v1.2.3") @@ -29,9 +28,9 @@ func TestVersion(t *testing.T) { func runVersionCommand(t *testing.T, commitID, tag string) string { orig := os.Stdout - defer func() {os.Stdout = orig}() + defer func() { os.Stdout = orig }() - r,w,e := os.Pipe() + r, w, e := os.Pipe() if e != nil { t.Error("Cannot setup pipes.") } @@ -41,8 +40,12 @@ func runVersionCommand(t *testing.T, commitID, tag string) string { // // needs to be set in the free wild by the build process: // go build -ldflags "-X github.com/SAP/jenkins-library/cmd.GitCommit=${GIT_COMMIT} -X github.com/SAP/jenkins-library/cmd.GitTag=${GIT_TAG}" - if len(commitID) > 0 { GitCommit = commitID; } - if len(tag) > 0 { GitTag = tag } + if len(commitID) > 0 { + GitCommit = commitID + } + if len(tag) > 0 { + GitTag = tag + } defer func() { GitCommit = ""; GitTag = "" }() // // @@ -58,4 +61,4 @@ func runVersionCommand(t *testing.T, commitID, tag string) string { var buf bytes.Buffer io.Copy(&buf, r) return buf.String() -} \ No newline at end of file +} diff --git a/resources/metadata/githubrelease.yaml b/resources/metadata/githubrelease.yaml index 07b0ae1ce..297dd7ead 100644 --- a/resources/metadata/githubrelease.yaml +++ b/resources/metadata/githubrelease.yaml @@ -101,6 +101,16 @@ spec: - STEPS type: string mandatory: true + - name: githubUploadUrl + description: Set the GitHub API url. + scope: + - GENERAL + - PARAMETERS + - STAGES + - STEPS + type: string + default: https://uploads.github.com + mandatory: true - name: labels description: 'Labels to include in issue search.' scope: @@ -115,8 +125,8 @@ spec: - STAGES - STEPS type: string - - name: update - description: Specify if the release should be updated in case it already exists + - name: updateAsset + description: Specify if a release asset should be updated only. scope: - PARAMETERS - STAGES From 44473666a884051a2b1d5f76466d3e6bd46021ed Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Tue, 5 Nov 2019 14:46:45 +0100 Subject: [PATCH 05/13] Address CodeClimate findings --- resources/metadata/githubrelease.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/metadata/githubrelease.yaml b/resources/metadata/githubrelease.yaml index 297dd7ead..580171334 100644 --- a/resources/metadata/githubrelease.yaml +++ b/resources/metadata/githubrelease.yaml @@ -4,13 +4,13 @@ metadata: longDescription: | This step creates a tag in your GitHub repository together with a release. The release can be filled with text plus additional information like: - + * Closed pull request since last release * Closed issues since last release * Link to delta information showing all commits since last release - + The result looks like - + ![Example release](../images/githubRelease.png) spec: inputs: From 1a034aea4cc2c370f3c5d344824a231a0e796e5d Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Tue, 5 Nov 2019 15:13:04 +0100 Subject: [PATCH 06/13] Update parameter names --- cmd/githubPublishRelease.go | 22 ++++++------ cmd/githubPublishRelease_generated.go | 48 +++++++++++++-------------- cmd/githubPublishRelease_test.go | 32 +++++++++--------- resources/metadata/githubrelease.yaml | 24 ++++++++++---- 4 files changed, 69 insertions(+), 57 deletions(-) diff --git a/cmd/githubPublishRelease.go b/cmd/githubPublishRelease.go index 701bdeb54..3d18f8e71 100644 --- a/cmd/githubPublishRelease.go +++ b/cmd/githubPublishRelease.go @@ -28,7 +28,7 @@ type githubIssueClient interface { } func githubPublishRelease(myGithubPublishReleaseOptions githubPublishReleaseOptions) error { - ctx, client, err := piperGithub.NewClient(myGithubPublishReleaseOptions.GithubToken, myGithubPublishReleaseOptions.GithubAPIURL, myGithubPublishReleaseOptions.GithubUploadURL) + ctx, client, err := piperGithub.NewClient(myGithubPublishReleaseOptions.Token, myGithubPublishReleaseOptions.ApiURL, myGithubPublishReleaseOptions.UploadURL) if err != nil { log.Entry().WithError(err).Fatal("Failed to get GitHub client.") } @@ -45,7 +45,7 @@ func runGithubPublishRelease(ctx context.Context, myGithubPublishReleaseOptions var publishedAt github.Timestamp - lastRelease, resp, err := ghRepoClient.GetLatestRelease(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo) + lastRelease, resp, err := ghRepoClient.GetLatestRelease(ctx, myGithubPublishReleaseOptions.Owner, myGithubPublishReleaseOptions.Repository) if err != nil { if resp.StatusCode == 404 { //no previous release found -> first release @@ -83,11 +83,11 @@ func runGithubPublishRelease(ctx context.Context, myGithubPublishReleaseOptions Body: &releaseBody, } - createdRelease, _, err := ghRepoClient.CreateRelease(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, &release) + createdRelease, _, err := ghRepoClient.CreateRelease(ctx, myGithubPublishReleaseOptions.Owner, myGithubPublishReleaseOptions.Repository, &release) if err != nil { return errors.Wrapf(err, "Creation of release '%v' failed", *release.TagName) } - log.Entry().Infof("Release %v created on %v/%v", *createdRelease.TagName, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo) + log.Entry().Infof("Release %v created on %v/%v", *createdRelease.TagName, myGithubPublishReleaseOptions.Owner, myGithubPublishReleaseOptions.Repository) if len(myGithubPublishReleaseOptions.AssetPath) > 0 { return uploadReleaseAsset(ctx, createdRelease.GetID(), myGithubPublishReleaseOptions, ghRepoClient) @@ -107,7 +107,7 @@ func getClosedIssuesText(ctx context.Context, publishedAt github.Timestamp, myGi if len(myGithubPublishReleaseOptions.Labels) > 0 { options.Labels = myGithubPublishReleaseOptions.Labels } - ghIssues, _, err := ghIssueClient.ListByRepo(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, &options) + ghIssues, _, err := ghIssueClient.ListByRepo(ctx, myGithubPublishReleaseOptions.Owner, myGithubPublishReleaseOptions.Repository, &options) if err != nil { log.Entry().WithError(err).Error("Failed to get GitHub issues.") } @@ -144,9 +144,9 @@ func getReleaseDeltaText(myGithubPublishReleaseOptions *githubPublishReleaseOpti "[%v...%v](%v/%v/%v/compare/%v...%v)\n", lastRelease.GetTagName(), myGithubPublishReleaseOptions.Version, - myGithubPublishReleaseOptions.GithubServerURL, - myGithubPublishReleaseOptions.GithubOrg, - myGithubPublishReleaseOptions.GithubRepo, + myGithubPublishReleaseOptions.ServerURL, + myGithubPublishReleaseOptions.Owner, + myGithubPublishReleaseOptions.Repository, lastRelease.GetTagName(), myGithubPublishReleaseOptions.Version, ) @@ -155,7 +155,7 @@ func getReleaseDeltaText(myGithubPublishReleaseOptions *githubPublishReleaseOpti func uploadReleaseAsset(ctx context.Context, releaseID int64, myGithubPublishReleaseOptions *githubPublishReleaseOptions, ghRepoClient githubRepoClient) error { - assets, _, err := ghRepoClient.ListReleaseAssets(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, releaseID, &github.ListOptions{}) + assets, _, err := ghRepoClient.ListReleaseAssets(ctx, myGithubPublishReleaseOptions.Owner, myGithubPublishReleaseOptions.Repository, releaseID, &github.ListOptions{}) if err != nil { return errors.Wrap(err, "Failed to get list of release assets.") } @@ -168,7 +168,7 @@ func uploadReleaseAsset(ctx context.Context, releaseID int64, myGithubPublishRel } if assetID != 0 { //asset needs to be deleted first since API does not allow for replacement - _, err := ghRepoClient.DeleteReleaseAsset(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, assetID) + _, err := ghRepoClient.DeleteReleaseAsset(ctx, myGithubPublishReleaseOptions.Owner, myGithubPublishReleaseOptions.Repository, assetID) if err != nil { return errors.Wrap(err, "Failed to delete release asset.") } @@ -194,7 +194,7 @@ func uploadReleaseAsset(ctx context.Context, releaseID int64, myGithubPublishRel } log.Entry().Info("Starting to upload release asset.") - asset, _, err := ghRepoClient.UploadReleaseAsset(ctx, myGithubPublishReleaseOptions.GithubOrg, myGithubPublishReleaseOptions.GithubRepo, releaseID, &opts, file) + asset, _, err := ghRepoClient.UploadReleaseAsset(ctx, myGithubPublishReleaseOptions.Owner, myGithubPublishReleaseOptions.Repository, releaseID, &opts, file) if err != nil { return errors.Wrap(err, "Failed to upload release asset.") } diff --git a/cmd/githubPublishRelease_generated.go b/cmd/githubPublishRelease_generated.go index 3fc00a1b3..9ba10ffae 100644 --- a/cmd/githubPublishRelease_generated.go +++ b/cmd/githubPublishRelease_generated.go @@ -14,12 +14,12 @@ type githubPublishReleaseOptions struct { AssetPath string `json:"assetPath,omitempty"` Commitish string `json:"commitish,omitempty"` ExcludeLabels []string `json:"excludeLabels,omitempty"` - GithubAPIURL string `json:"githubApiUrl,omitempty"` - GithubOrg string `json:"githubOrg,omitempty"` - GithubRepo string `json:"githubRepo,omitempty"` - GithubServerURL string `json:"githubServerUrl,omitempty"` - GithubToken string `json:"githubToken,omitempty"` - GithubUploadURL string `json:"githubUploadUrl,omitempty"` + ApiURL string `json:"apiUrl,omitempty"` + Owner string `json:"owner,omitempty"` + Repository string `json:"repository,omitempty"` + ServerURL string `json:"serverUrl,omitempty"` + Token string `json:"token,omitempty"` + UploadURL string `json:"uploadUrl,omitempty"` Labels []string `json:"labels,omitempty"` ReleaseBodyHeader string `json:"releaseBodyHeader,omitempty"` UpdateAsset bool `json:"updateAsset,omitempty"` @@ -65,23 +65,23 @@ func addGithubPublishReleaseFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&myGithubPublishReleaseOptions.AssetPath, "assetPath", os.Getenv("PIPER_assetPath"), "Path to a release asset which should be uploaded to the list of release assets.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Commitish, "commitish", "master", "Target git commitish for the release") cmd.Flags().StringSliceVar(&myGithubPublishReleaseOptions.ExcludeLabels, "excludeLabels", []string{}, "Allows to exclude issues with dedicated list of labels.") - cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubAPIURL, "githubApiUrl", "https://api.github.com", "Set the GitHub API url.") - cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubOrg, "githubOrg", os.Getenv("PIPER_githubOrg"), "Set the GitHub organization.") - cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubRepo, "githubRepo", os.Getenv("PIPER_githubRepo"), "Set the GitHub repository.") - cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubServerURL, "githubServerUrl", "https://github.com", "GitHub server url for end-user access.") - cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubToken, "githubToken", os.Getenv("PIPER_githubToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") - cmd.Flags().StringVar(&myGithubPublishReleaseOptions.GithubUploadURL, "githubUploadUrl", "https://uploads.github.com", "Set the GitHub API url.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.ApiURL, "apiUrl", "https://api.github.com", "Set the GitHub API url.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Owner, "owner", os.Getenv("PIPER_owner"), "Set the GitHub organization.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Repository, "repository", os.Getenv("PIPER_repository"), "Set the GitHub repository.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.ServerURL, "serverUrl", "https://github.com", "GitHub server url for end-user access.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Token, "token", os.Getenv("PIPER_token"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.UploadURL, "uploadUrl", "https://uploads.github.com", "Set the GitHub API url.") cmd.Flags().StringSliceVar(&myGithubPublishReleaseOptions.Labels, "labels", []string{}, "Labels to include in issue search.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.ReleaseBodyHeader, "releaseBodyHeader", os.Getenv("PIPER_releaseBodyHeader"), "Content which will appear for the release.") cmd.Flags().BoolVar(&myGithubPublishReleaseOptions.UpdateAsset, "updateAsset", false, "Specify if a release asset should be updated only.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Version, "version", os.Getenv("PIPER_version"), "Define the version number which will be written as tag as well as release name.") - cmd.MarkFlagRequired("githubApiUrl") - cmd.MarkFlagRequired("githubOrg") - cmd.MarkFlagRequired("githubRepo") - cmd.MarkFlagRequired("githubServerUrl") - cmd.MarkFlagRequired("githubToken") - cmd.MarkFlagRequired("githubUploadUrl") + cmd.MarkFlagRequired("apiUrl") + cmd.MarkFlagRequired("owner") + cmd.MarkFlagRequired("repository") + cmd.MarkFlagRequired("serverUrl") + cmd.MarkFlagRequired("token") + cmd.MarkFlagRequired("uploadUrl") cmd.MarkFlagRequired("version") } @@ -122,37 +122,37 @@ func githubPublishReleaseMetadata() config.StepData { Mandatory: false, }, { - Name: "githubApiUrl", + Name: "apiUrl", Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, }, { - Name: "githubOrg", + Name: "owner", Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, }, { - Name: "githubRepo", + Name: "repository", Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, }, { - Name: "githubServerUrl", + Name: "serverUrl", Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, }, { - Name: "githubToken", + Name: "token", Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, }, { - Name: "githubUploadUrl", + Name: "uploadUrl", Scope: []string{"GENERAL", "PARAMETERS", "STAGES", "STEPS"}, Type: "string", Mandatory: true, diff --git a/cmd/githubPublishRelease_test.go b/cmd/githubPublishRelease_test.go index 88ff54ec0..9008d3e8c 100644 --- a/cmd/githubPublishRelease_test.go +++ b/cmd/githubPublishRelease_test.go @@ -101,9 +101,9 @@ func TestRunGithubPublishRelease(t *testing.T) { myGithubPublishReleaseOptions := githubPublishReleaseOptions{ AddDeltaToLastRelease: true, Commitish: "master", - GithubOrg: "TEST", - GithubRepo: "test", - GithubServerURL: "https://github.com", + Owner: "TEST", + Repository: "test", + ServerURL: "https://github.com", ReleaseBodyHeader: "Header", Version: "1.0", } @@ -141,9 +141,9 @@ func TestRunGithubPublishRelease(t *testing.T) { AddClosedIssues: true, AddDeltaToLastRelease: true, Commitish: "master", - GithubOrg: "TEST", - GithubRepo: "test", - GithubServerURL: "https://github.com", + Owner: "TEST", + Repository: "test", + ServerURL: "https://github.com", ReleaseBodyHeader: "Header", Version: "1.1", } @@ -245,8 +245,8 @@ func TestGetClosedIssuesText(t *testing.T) { } myGithubPublishReleaseOptions := githubPublishReleaseOptions{ - GithubOrg: "TEST", - GithubRepo: "test", + Owner: "TEST", + Repository: "test", } res := getClosedIssuesText(ctx, publishedAt, &myGithubPublishReleaseOptions, &ghIssueClient) @@ -263,10 +263,10 @@ func TestGetClosedIssuesText(t *testing.T) { func TestGetReleaseDeltaText(t *testing.T) { myGithubPublishReleaseOptions := githubPublishReleaseOptions{ - GithubOrg: "TEST", - GithubRepo: "test", - GithubServerURL: "https://github.com", - Version: "1.1", + Owner: "TEST", + Repository: "test", + ServerURL: "https://github.com", + Version: "1.1", } lastTag := "1.0" lastRelease := github.RepositoryRelease{ @@ -295,8 +295,8 @@ func TestUploadReleaseAsset(t *testing.T) { } myGithubPublishReleaseOptions := githubPublishReleaseOptions{ - GithubOrg: "TEST", - GithubRepo: "test", + Owner: "TEST", + Repository: "test", AssetPath: filepath.Join("testdata", t.Name()+"_test.txt"), } @@ -332,8 +332,8 @@ func TestUploadReleaseAsset(t *testing.T) { } myGithubPublishReleaseOptions := githubPublishReleaseOptions{ - GithubOrg: "TEST", - GithubRepo: "test", + Owner: "TEST", + Repository: "test", AssetPath: filepath.Join("testdata", t.Name()+"_test.txt"), } diff --git a/resources/metadata/githubrelease.yaml b/resources/metadata/githubrelease.yaml index 580171334..8ed7b4d23 100644 --- a/resources/metadata/githubrelease.yaml +++ b/resources/metadata/githubrelease.yaml @@ -57,7 +57,9 @@ spec: - STAGES - STEPS type: '[]string' - - name: githubApiUrl + - name: apiUrl + aliases: + - name: githubApiUrl description: Set the GitHub API url. scope: - GENERAL @@ -67,7 +69,9 @@ spec: type: string default: https://api.github.com mandatory: true - - name: githubOrg + - name: owner + aliases: + - name: githubOrg description: 'Set the GitHub organization.' scope: - PARAMETERS @@ -75,7 +79,9 @@ spec: - STEPS type: string mandatory: true - - name: githubRepo + - name: repository + aliases: + - name: githubRepo description: 'Set the GitHub repository.' scope: - PARAMETERS @@ -83,7 +89,9 @@ spec: - STEPS type: string mandatory: true - - name: githubServerUrl + - name: serverUrl + aliases: + - name: githubServerUrl description: 'GitHub server url for end-user access.' scope: - PARAMETERS @@ -92,7 +100,9 @@ spec: type: string default: https://github.com mandatory: true - - name: githubToken + - name: token + aliases: + - name: githubToken description: 'GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line' scope: - GENERAL @@ -101,7 +111,9 @@ spec: - STEPS type: string mandatory: true - - name: githubUploadUrl + - name: uploadUrl + aliases: + - name: githubUploadUrl description: Set the GitHub API url. scope: - GENERAL From 376419e0dbfb047895708c8201e51c420394e320 Mon Sep 17 00:00:00 2001 From: Sven Merk Date: Tue, 5 Nov 2019 16:30:41 +0100 Subject: [PATCH 07/13] Add mixin of dependent defaults --- .gitignore | 1 + go.mod | 1 + go.sum | 2 ++ pkg/config/config.go | 19 +++++++++++++- pkg/config/config_test.go | 52 +++++++++++++++++++++++++++++++-------- 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 8225066ba..570cfac54 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ reports .classpath .project *~ +.vscode # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* diff --git a/go.mod b/go.mod index 2c87f65d7..dd84712ba 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( github.com/ghodss/yaml v1.0.0 + github.com/google/go-cmp v0.3.1 github.com/pkg/errors v0.8.1 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 796e8ee0a..e65e99c2c 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= diff --git a/pkg/config/config.go b/pkg/config/config.go index 622e9dffb..bc1bfce22 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/ghodss/yaml" + "github.com/google/go-cmp/cmp" "github.com/pkg/errors" ) @@ -133,6 +134,22 @@ func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON stri stepConfig.mixIn(flagValues, filters.Parameters) } + // finally do the condition evaluation post processing + for _, p := range parameters { + if len(p.Conditions) > 0 { + cp := p.Conditions[0].Params[0] + dependentValue := stepConfig.Config[cp.Name] + if cmp.Equal(dependentValue, cp.Value) && stepConfig.Config[p.Name] == nil { + subMapValue := stepConfig.Config[dependentValue.(string)].(map[string]interface{})[p.Name] + if subMapValue != nil { + stepConfig.Config[p.Name] = subMapValue + } else { + stepConfig.Config[p.Name] = p.Default + } + } + } + } + return stepConfig, nil } @@ -180,7 +197,7 @@ func (s *StepConfig) mixIn(mergeData map[string]interface{}, filter []string) { s.Config = map[string]interface{}{} } - s.Config = filterMap(merge(s.Config, mergeData), filter) + s.Config = merge(s.Config, filterMap(mergeData, filter)) } func filterMap(data map[string]interface{}, filter []string) map[string]interface{} { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 3c81bad76..6d1ea733e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -74,6 +74,7 @@ steps: p4: p4_step px4: px4_step p5: p5_step + dependentParameter: dependentValue stages: stage1: p5: p5_stage @@ -82,7 +83,7 @@ stages: ` filters := StepFilters{ General: []string{"p0", "p1", "p2", "p3", "p4"}, - Steps: []string{"p0", "p1", "p2", "p3", "p4", "p5"}, + Steps: []string{"p0", "p1", "p2", "p3", "p4", "p5", "dependentParameter", "pd1", "dependentValue", "pd2"}, Stages: []string{"p0", "p1", "p2", "p3", "p4", "p5", "p6"}, Parameters: []string{"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7"}, Env: []string{"p0", "p1", "p2", "p3", "p4", "p5"}, @@ -97,6 +98,8 @@ steps: p1: p1_step_default px1: px1_step_default p2: p2_step_default + dependentValue: + pd1: pd1_dependent_default ` defaults2 := `general: @@ -112,20 +115,49 @@ steps: defaults := []io.ReadCloser{ioutil.NopCloser(strings.NewReader(defaults1)), ioutil.NopCloser(strings.NewReader(defaults2))} myConfig := ioutil.NopCloser(strings.NewReader(testConfig)) - stepConfig, err := c.GetStepConfig(flags, paramJSON, myConfig, defaults, filters, []StepParameters{}, "stage1", "step1") + + parameterMetadata := []StepParameters{ + { + Name: "pd1", + Scope: []string{"STEPS"}, + Conditions: []Condition{ + { + Params: []Param{ + {Name: "dependentParameter", Value: "dependentValue"}, + }, + }, + }, + }, + { + Name: "pd2", + Default: "pd2_metadata_default", + Scope: []string{"STEPS"}, + Conditions: []Condition{ + { + Params: []Param{ + {Name: "dependentParameter", Value: "dependentValue"}, + }, + }, + }, + }, + } + + stepConfig, err := c.GetStepConfig(flags, paramJSON, myConfig, defaults, filters, parameterMetadata, "stage1", "step1") assert.Equal(t, nil, err, "error occured but none expected") t.Run("Config", func(t *testing.T) { expected := map[string]string{ - "p0": "p0_general_default", - "p1": "p1_step_default", - "p2": "p2_general_default", - "p3": "p3_general", - "p4": "p4_step", - "p5": "p5_stage", - "p6": "p6_param", - "p7": "p7_flag", + "p0": "p0_general_default", + "p1": "p1_step_default", + "p2": "p2_general_default", + "p3": "p3_general", + "p4": "p4_step", + "p5": "p5_stage", + "p6": "p6_param", + "p7": "p7_flag", + "pd1": "pd1_dependent_default", + "pd2": "pd2_metadata_default", } for k, v := range expected { t.Run(k, func(t *testing.T) { From 8587452a3c1c6da46fd03200d51e7cc18859d1dd Mon Sep 17 00:00:00 2001 From: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Date: Tue, 5 Nov 2019 17:29:05 +0100 Subject: [PATCH 08/13] Update resources/metadata/githubrelease.yaml Co-Authored-By: Christopher Fenner --- resources/metadata/githubrelease.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/metadata/githubrelease.yaml b/resources/metadata/githubrelease.yaml index 8ed7b4d23..6e8cefa55 100644 --- a/resources/metadata/githubrelease.yaml +++ b/resources/metadata/githubrelease.yaml @@ -90,7 +90,7 @@ spec: type: string mandatory: true - name: serverUrl - aliases: + aliases: - name: githubServerUrl description: 'GitHub server url for end-user access.' scope: From 48bfc10956586f925ce2ade724ba0462417a9f10 Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Tue, 5 Nov 2019 17:33:00 +0100 Subject: [PATCH 09/13] Address PR feedback --- cmd/githubPublishRelease.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/githubPublishRelease.go b/cmd/githubPublishRelease.go index 3d18f8e71..1963bda85 100644 --- a/cmd/githubPublishRelease.go +++ b/cmd/githubPublishRelease.go @@ -112,8 +112,8 @@ func getClosedIssuesText(ctx context.Context, publishedAt github.Timestamp, myGi log.Entry().WithError(err).Error("Failed to get GitHub issues.") } - prTexts := []string{"\n**List of closed pull-requests since last release**"} - issueTexts := []string{"\n**List of closed issues since last release**"} + prTexts := []string{"**List of closed pull-requests since last release**"} + issueTexts := []string{"**List of closed issues since last release**"} for _, issue := range ghIssues { if issue.IsPullRequest() && !isExcluded(issue, myGithubPublishReleaseOptions.ExcludeLabels) { @@ -126,11 +126,11 @@ func getClosedIssuesText(ctx context.Context, publishedAt github.Timestamp, myGi } if len(prTexts) > 1 { - closedIssuesText += strings.Join(prTexts, "\n") + "\n" + closedIssuesText += "\n" + strings.Join(prTexts, "\n") + "\n" } if len(issueTexts) > 1 { - closedIssuesText += strings.Join(issueTexts, "\n") + "\n" + closedIssuesText += "\n" + strings.Join(issueTexts, "\n") + "\n" } return closedIssuesText } From 5c87d4775ca355245f99177da242f702098507c0 Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Wed, 6 Nov 2019 09:05:07 +0100 Subject: [PATCH 10/13] Ensure asset update for latest release only --- cmd/githubPublishRelease.go | 3 ++- cmd/githubPublishRelease_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/githubPublishRelease.go b/cmd/githubPublishRelease.go index 1963bda85..aad18dd4d 100644 --- a/cmd/githubPublishRelease.go +++ b/cmd/githubPublishRelease.go @@ -58,7 +58,8 @@ func runGithubPublishRelease(ctx context.Context, myGithubPublishReleaseOptions publishedAt = lastRelease.GetPublishedAt() log.Entry().Debugf("Previous GitHub release published: '%v'", publishedAt) - if myGithubPublishReleaseOptions.UpdateAsset { + //updating assets only supported on latest release + if myGithubPublishReleaseOptions.UpdateAsset && myGithubPublishReleaseOptions.Version == "latest" { return uploadReleaseAsset(ctx, lastRelease.GetID(), myGithubPublishReleaseOptions, ghRepoClient) } diff --git a/cmd/githubPublishRelease_test.go b/cmd/githubPublishRelease_test.go index 9008d3e8c..95aafc493 100644 --- a/cmd/githubPublishRelease_test.go +++ b/cmd/githubPublishRelease_test.go @@ -171,6 +171,7 @@ func TestRunGithubPublishRelease(t *testing.T) { myGithubPublishReleaseOptions := githubPublishReleaseOptions{ UpdateAsset: true, AssetPath: filepath.Join("testdata", t.Name()+"_test.txt"), + Version: "latest", } err := runGithubPublishRelease(ctx, &myGithubPublishReleaseOptions, &ghRepoClient, &ghIssueClient) From deb965e2b418ba5f72a64b0f4ee5a118091bad0a Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Wed, 6 Nov 2019 09:12:50 +0100 Subject: [PATCH 11/13] Fix CodeClimate finding with generator update --- cmd/githubPublishRelease.go | 2 +- cmd/githubPublishRelease_generated.go | 4 ++-- pkg/generator/step-metadata.go | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/githubPublishRelease.go b/cmd/githubPublishRelease.go index aad18dd4d..a7f70e471 100644 --- a/cmd/githubPublishRelease.go +++ b/cmd/githubPublishRelease.go @@ -28,7 +28,7 @@ type githubIssueClient interface { } func githubPublishRelease(myGithubPublishReleaseOptions githubPublishReleaseOptions) error { - ctx, client, err := piperGithub.NewClient(myGithubPublishReleaseOptions.Token, myGithubPublishReleaseOptions.ApiURL, myGithubPublishReleaseOptions.UploadURL) + ctx, client, err := piperGithub.NewClient(myGithubPublishReleaseOptions.Token, myGithubPublishReleaseOptions.APIURL, myGithubPublishReleaseOptions.UploadURL) if err != nil { log.Entry().WithError(err).Fatal("Failed to get GitHub client.") } diff --git a/cmd/githubPublishRelease_generated.go b/cmd/githubPublishRelease_generated.go index 9ba10ffae..faddfaf2e 100644 --- a/cmd/githubPublishRelease_generated.go +++ b/cmd/githubPublishRelease_generated.go @@ -14,7 +14,7 @@ type githubPublishReleaseOptions struct { AssetPath string `json:"assetPath,omitempty"` Commitish string `json:"commitish,omitempty"` ExcludeLabels []string `json:"excludeLabels,omitempty"` - ApiURL string `json:"apiUrl,omitempty"` + APIURL string `json:"apiUrl,omitempty"` Owner string `json:"owner,omitempty"` Repository string `json:"repository,omitempty"` ServerURL string `json:"serverUrl,omitempty"` @@ -65,7 +65,7 @@ func addGithubPublishReleaseFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&myGithubPublishReleaseOptions.AssetPath, "assetPath", os.Getenv("PIPER_assetPath"), "Path to a release asset which should be uploaded to the list of release assets.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Commitish, "commitish", "master", "Target git commitish for the release") cmd.Flags().StringSliceVar(&myGithubPublishReleaseOptions.ExcludeLabels, "excludeLabels", []string{}, "Allows to exclude issues with dedicated list of labels.") - cmd.Flags().StringVar(&myGithubPublishReleaseOptions.ApiURL, "apiUrl", "https://api.github.com", "Set the GitHub API url.") + cmd.Flags().StringVar(&myGithubPublishReleaseOptions.APIURL, "apiUrl", "https://api.github.com", "Set the GitHub API url.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Owner, "owner", os.Getenv("PIPER_owner"), "Set the GitHub organization.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.Repository, "repository", os.Getenv("PIPER_repository"), "Set the GitHub repository.") cmd.Flags().StringVar(&myGithubPublishReleaseOptions.ServerURL, "serverUrl", "https://github.com", "GitHub server url for end-user access.") diff --git a/pkg/generator/step-metadata.go b/pkg/generator/step-metadata.go index 3a056e09d..fa46fe994 100644 --- a/pkg/generator/step-metadata.go +++ b/pkg/generator/step-metadata.go @@ -295,6 +295,7 @@ func longName(long string) string { func golangName(name string) string { properName := strings.Replace(name, "Api", "API", -1) + properName = strings.Replace(name, "api", "API", -1) properName = strings.Replace(properName, "Url", "URL", -1) properName = strings.Replace(properName, "Id", "ID", -1) properName = strings.Replace(properName, "Json", "JSON", -1) From 57540d31279d1224fb57804edd376807cf50fb77 Mon Sep 17 00:00:00 2001 From: OliverNocon Date: Wed, 6 Nov 2019 10:32:02 +0100 Subject: [PATCH 12/13] Fix bug in replacement function --- pkg/generator/step-metadata.go | 2 +- pkg/generator/step-metadata_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/generator/step-metadata.go b/pkg/generator/step-metadata.go index fa46fe994..fabf35c74 100644 --- a/pkg/generator/step-metadata.go +++ b/pkg/generator/step-metadata.go @@ -295,7 +295,7 @@ func longName(long string) string { func golangName(name string) string { properName := strings.Replace(name, "Api", "API", -1) - properName = strings.Replace(name, "api", "API", -1) + properName = strings.Replace(properName, "api", "API", -1) properName = strings.Replace(properName, "Url", "URL", -1) properName = strings.Replace(properName, "Id", "ID", -1) properName = strings.Replace(properName, "Json", "JSON", -1) diff --git a/pkg/generator/step-metadata_test.go b/pkg/generator/step-metadata_test.go index 2af975ca7..0285b0360 100644 --- a/pkg/generator/step-metadata_test.go +++ b/pkg/generator/step-metadata_test.go @@ -203,6 +203,7 @@ func TestGolangName(t *testing.T) { expected string }{ {input: "testApi", expected: "TestAPI"}, + {input: "apiTest", expected: "APITest"}, {input: "testUrl", expected: "TestURL"}, {input: "testId", expected: "TestID"}, {input: "testJson", expected: "TestJSON"}, From de31cde9b82e80334a9789ce6722827d2fe29893 Mon Sep 17 00:00:00 2001 From: Oliver Nocon <33484802+OliverNocon@users.noreply.github.com> Date: Wed, 6 Nov 2019 11:28:10 +0100 Subject: [PATCH 13/13] Add PiperGoUtils for downloading piper binary (#928) * Add PiperGoUtils for downloading piper binary PiperGoUtils provide the link between a Jenkins library step and the library step execution running in a go binary. It makes sure that an adequate binary is available. * fix CodeClimate finding * Remove Delimiter and add download resilience. --- .gitignore | 4 +- go.sum | 1 + src/com/sap/piper/JenkinsUtils.groovy | 19 +++ src/com/sap/piper/PiperGoUtils.groovy | 63 +++++++++ .../com/sap/piper/JenkinsUtilsTest.groovy | 18 +++ .../com/sap/piper/PiperGoUtilsTest.groovy | 129 ++++++++++++++++++ 6 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 src/com/sap/piper/PiperGoUtils.groovy create mode 100644 test/groovy/com/sap/piper/PiperGoUtilsTest.groovy diff --git a/.gitignore b/.gitignore index 8225066ba..98be8170b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,5 +19,5 @@ documentation/docs-gen consumer-test/**/workspace *.code-workspace -piper -piper.exe +/piper +/piper.exe diff --git a/go.sum b/go.sum index 138315660..23b2c9ede 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= diff --git a/src/com/sap/piper/JenkinsUtils.groovy b/src/com/sap/piper/JenkinsUtils.groovy index 58296f7a9..6b43ffa77 100644 --- a/src/com/sap/piper/JenkinsUtils.groovy +++ b/src/com/sap/piper/JenkinsUtils.groovy @@ -7,6 +7,7 @@ import hudson.tasks.junit.TestResultAction import jenkins.model.Jenkins import org.apache.commons.io.IOUtils +import org.jenkinsci.plugins.workflow.libs.LibrariesAction import org.jenkinsci.plugins.workflow.steps.MissingContextVariableException @API @@ -108,3 +109,21 @@ String getIssueCommentTriggerAction() { def getJobStartedByUserId() { return getRawBuild().getCause(hudson.model.Cause.UserIdCause.class)?.getUserId() } + +@NonCPS +def getLibrariesInfo() { + def libraries = [] + def build = getRawBuild() + def libs = build.getAction(LibrariesAction.class).getLibraries() + + for (def i = 0; i < libs.size(); i++) { + Map lib = [:] + + lib['name'] = libs[i].name + lib['version'] = libs[i].version + lib['trusted'] = libs[i].trusted + libraries.add(lib) + } + + return libraries +} diff --git a/src/com/sap/piper/PiperGoUtils.groovy b/src/com/sap/piper/PiperGoUtils.groovy new file mode 100644 index 000000000..c575433cb --- /dev/null +++ b/src/com/sap/piper/PiperGoUtils.groovy @@ -0,0 +1,63 @@ +package com.sap.piper + +class PiperGoUtils implements Serializable { + + private static Script steps + private static Utils utils + + PiperGoUtils(Script steps) { + this.steps = steps + this.utils = new Utils() + } + + PiperGoUtils(Script steps, Utils utils) { + this.steps = steps + this.utils = utils + } + + void unstashPiperBin() { + + if (utils.unstash('piper-bin').size() > 0) return + + def libraries = getLibrariesInfo() + String version + libraries.each {lib -> + if (lib.name == 'piper-lib-os') { + version = lib.version + } + } + + def fallbackUrl = 'https://github.com/SAP/jenkins-library/releases/latest/download/piper_master' + def piperBinUrl = (version == 'master') ? fallbackUrl : "https://github.com/SAP/jenkins-library/releases/tag/${version}" + + boolean downloaded = downloadGoBinary(piperBinUrl) + if (!downloaded) { + //Inform that no Piper binary is available for used library branch + steps.echo ("Not able to download go binary of Piper for version ${version}") + //Fallback to master version & throw error in case this fails + steps.retry(5) { + if (!downloadGoBinary(fallbackUrl)) { + steps.sleep(2) + steps.error("Download of Piper go binary failed.") + } + } + + } + utils.stashWithMessage('piper-bin', 'failed to stash piper binary', 'piper') + } + + List getLibrariesInfo() { + return new JenkinsUtils().getLibrariesInfo() + } + + private boolean downloadGoBinary(url) { + + def httpStatus = steps.sh(returnStdout: true, script: "curl --insecure --silent --location --write-out '%{http_code}' --output ./piper '${url}'") + + if (httpStatus == '200') { + steps.sh(script: 'chmod +x ./piper') + return true + } + return false + } +} diff --git a/test/groovy/com/sap/piper/JenkinsUtilsTest.groovy b/test/groovy/com/sap/piper/JenkinsUtilsTest.groovy index b915bc82e..b07fe41b8 100644 --- a/test/groovy/com/sap/piper/JenkinsUtilsTest.groovy +++ b/test/groovy/com/sap/piper/JenkinsUtilsTest.groovy @@ -71,6 +71,16 @@ class JenkinsUtilsTest extends BasePiperTest { return triggerCause } } + def getAction(type) { + return new Object() { + def getLibraries() { + return [ + [name: 'lib1', version: '1', trusted: true], + [name: 'lib2', version: '2', trusted: false], + ] + } + } + } } LibraryLoadingTestExecutionListener.prepareObjectInterceptors(rawBuildMock) @@ -130,4 +140,12 @@ class JenkinsUtilsTest extends BasePiperTest { userId = null assertThat(jenkinsUtils.getJobStartedByUserId(), isEmptyOrNullString()) } + + @Test + void testGetLibrariesInfo() { + def libs + libs = jenkinsUtils.getLibrariesInfo() + assertThat(libs[0], is([name: 'lib1', version: '1', trusted: true])) + assertThat(libs[1], is([name: 'lib2', version: '2', trusted: false])) + } } diff --git a/test/groovy/com/sap/piper/PiperGoUtilsTest.groovy b/test/groovy/com/sap/piper/PiperGoUtilsTest.groovy new file mode 100644 index 000000000..92f64b370 --- /dev/null +++ b/test/groovy/com/sap/piper/PiperGoUtilsTest.groovy @@ -0,0 +1,129 @@ +package com.sap.piper + +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.ExpectedException +import org.junit.rules.RuleChain +import util.BasePiperTest +import util.JenkinsLoggingRule +import util.JenkinsShellCallRule +import util.Rules + +import static org.hamcrest.Matchers.containsString +import static org.hamcrest.Matchers.is +import static org.junit.Assert.assertThat + +class PiperGoUtilsTest extends BasePiperTest { + + public ExpectedException exception = ExpectedException.none() + public JenkinsShellCallRule shellCallRule = new JenkinsShellCallRule(this) + public JenkinsLoggingRule loggingRule = new JenkinsLoggingRule(this) + + @Rule + public RuleChain ruleChain = Rules.getCommonRules(this) + .around(shellCallRule) + .around(exception) + .around(loggingRule) + + @Before + void init() { + helper.registerAllowedMethod("retry", [Integer, Closure], null) + } + + @Test + void testUnstashPiperBinAvailable() { + + def piperBinStash = 'piper-bin' + + // this mocks utils.unstash + helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> + if (stashFileName != piperBinStash) { + return [] + } + return [piperBinStash] + }) + + def piperGoUtils = new PiperGoUtils(nullScript, utils) + + piperGoUtils.unstashPiperBin() + } + + + @Test + void testUnstashPiperBinMaster() { + + def piperGoUtils = new PiperGoUtils(nullScript, utils) + piperGoUtils.metaClass.getLibrariesInfo = {-> return [[name: 'piper-lib-os', version: 'master']]} + + // this mocks utils.unstash - mimic stash not existing + helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> + return [] + }) + + shellCallRule.setReturnValue('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper_master\'', '200') + + piperGoUtils.unstashPiperBin() + assertThat(shellCallRule.shell.size(), is(2)) + assertThat(shellCallRule.shell[0].toString(), is('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper_master\'')) + assertThat(shellCallRule.shell[1].toString(), is('chmod +x ./piper')) + } + + @Test + void testUnstashPiperBinNonMaster() { + + def piperGoUtils = new PiperGoUtils(nullScript, utils) + piperGoUtils.metaClass.getLibrariesInfo = {-> return [[name: 'piper-lib-os', version: 'testTag']]} + + // this mocks utils.unstash - mimic stash not existing + helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> + return [] + }) + + shellCallRule.setReturnValue('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/tag/testTag\'', '200') + + piperGoUtils.unstashPiperBin() + assertThat(shellCallRule.shell.size(), is(2)) + assertThat(shellCallRule.shell[0].toString(), is('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/tag/testTag\'')) + assertThat(shellCallRule.shell[1].toString(), is('chmod +x ./piper')) + } + + @Test + void testUnstashPiperBinFallback() { + + def piperGoUtils = new PiperGoUtils(nullScript, utils) + piperGoUtils.metaClass.getLibrariesInfo = {-> return [[name: 'piper-lib-os', version: 'notAvailable']]} + + shellCallRule.setReturnValue('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/tag/notAvailable\'', '404') + shellCallRule.setReturnValue('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper_master\'', '200') + + // this mocks utils.unstash - mimic stash not existing + helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> + return [] + }) + + piperGoUtils.unstashPiperBin() + assertThat(shellCallRule.shell.size(), is(3)) + assertThat(shellCallRule.shell[0].toString(), is('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/tag/notAvailable\'')) + assertThat(shellCallRule.shell[1].toString(), is('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper_master\'')) + assertThat(shellCallRule.shell[2].toString(), is('chmod +x ./piper')) + } + + @Test + void testDownloadFailed() { + def piperGoUtils = new PiperGoUtils(nullScript, utils) + piperGoUtils.metaClass.getLibrariesInfo = {-> return [[name: 'piper-lib-os', version: 'notAvailable']]} + + shellCallRule.setReturnValue('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/tag/notAvailable\'', '404') + shellCallRule.setReturnValue('curl --insecure --silent --location --write-out \'%{http_code}\' --output ./piper \'https://github.com/SAP/jenkins-library/releases/latest/download/piper_master\'', '500') + + helper.registerAllowedMethod("unstash", [String.class], { stashFileName -> + return [] + }) + + exception.expectMessage(containsString('Download of Piper go binary failed')) + piperGoUtils.unstashPiperBin() + } +} + +