mirror of
https://github.com/SAP/jenkins-library.git
synced 2024-12-12 10:55:20 +02:00
feat(cnbBuild): added support for project.toml (#3163)
Co-authored-by: Sumit Kulhadia <sumit.kulhadia@sap.com> Co-authored-by: Johannes Dillmann <j.dillmann@sap.com>
This commit is contained in:
Pavel Busko
GitHub
parent
f585e932ef
commit
372cef04b4
@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/SAP/jenkins-library/pkg/certutils"
|
||||
"github.com/SAP/jenkins-library/pkg/cnbutils"
|
||||
"github.com/SAP/jenkins-library/pkg/cnbutils/project"
|
||||
"github.com/SAP/jenkins-library/pkg/command"
|
||||
"github.com/SAP/jenkins-library/pkg/docker"
|
||||
piperhttp "github.com/SAP/jenkins-library/pkg/http"
|
||||
@ -20,6 +21,7 @@ import (
|
||||
"github.com/SAP/jenkins-library/pkg/telemetry"
|
||||
"github.com/docker/cli/cli/config/configfile"
|
||||
"github.com/pkg/errors"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -72,8 +74,29 @@ func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commo
|
||||
}
|
||||
}
|
||||
|
||||
func isIgnored(find string) bool {
|
||||
return strings.HasSuffix(find, "piper") || strings.Contains(find, ".pipeline")
|
||||
func isIgnored(find string, include, exclude *ignore.GitIgnore) bool {
|
||||
if exclude != nil {
|
||||
filtered := exclude.MatchesPath(find)
|
||||
|
||||
if filtered {
|
||||
log.Entry().Debugf("%s matches exclude pattern, ignoring", find)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if include != nil {
|
||||
filtered := !include.MatchesPath(find)
|
||||
|
||||
if filtered {
|
||||
log.Entry().Debugf("%s doesn't match include pattern, ignoring", find)
|
||||
return true
|
||||
} else {
|
||||
log.Entry().Debugf("%s matches include pattern", find)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func isDir(path string) (bool, error) {
|
||||
@ -108,10 +131,30 @@ func isZip(path string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
func copyProject(source, target string, utils cnbutils.BuildUtils) error {
|
||||
func copyFile(source, target string, utils cnbutils.BuildUtils) error {
|
||||
targetDir := filepath.Dir(target)
|
||||
|
||||
exists, err := utils.DirExists(targetDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !exists {
|
||||
log.Entry().Debugf("Creating directory %s", targetDir)
|
||||
err = utils.MkdirAll(targetDir, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = utils.Copy(source, target)
|
||||
return err
|
||||
}
|
||||
|
||||
func copyProject(source, target string, include, exclude *ignore.GitIgnore, utils cnbutils.BuildUtils) error {
|
||||
sourceFiles, _ := utils.Glob(path.Join(source, "**"))
|
||||
for _, sourceFile := range sourceFiles {
|
||||
if !isIgnored(sourceFile) {
|
||||
if !isIgnored(sourceFile, include, exclude) {
|
||||
target := path.Join(target, strings.ReplaceAll(sourceFile, source, ""))
|
||||
dir, err := isDir(sourceFile)
|
||||
if err != nil {
|
||||
@ -127,21 +170,19 @@ func copyProject(source, target string, utils cnbutils.BuildUtils) error {
|
||||
}
|
||||
} else {
|
||||
log.Entry().Debugf("Copying '%s' to '%s'", sourceFile, target)
|
||||
_, err = utils.Copy(sourceFile, target)
|
||||
err = copyFile(sourceFile, target, utils)
|
||||
if err != nil {
|
||||
log.SetErrorCategory(log.ErrorBuild)
|
||||
return errors.Wrapf(err, "Copying '%s' to '%s' failed", sourceFile, target)
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
log.Entry().Debugf("Filtered out '%s'", sourceFile)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyFile(source, target string, utils cnbutils.BuildUtils) error {
|
||||
func extractZip(source, target string, utils cnbutils.BuildUtils) error {
|
||||
|
||||
if isZip(source) {
|
||||
log.Entry().Infof("Extracting archive '%s' to '%s'", source, target)
|
||||
@ -181,13 +222,46 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, u
|
||||
|
||||
if err != nil {
|
||||
log.SetErrorCategory(log.ErrorConfiguration)
|
||||
return errors.Wrapf(err, "failed to check if dockerImage is a valid builder")
|
||||
return errors.Wrap(err, "failed to check if dockerImage is a valid builder")
|
||||
}
|
||||
if !exists {
|
||||
log.SetErrorCategory(log.ErrorConfiguration)
|
||||
return errors.New("the provided dockerImage is not a valid builder")
|
||||
}
|
||||
|
||||
include := ignore.CompileIgnoreLines("**/*")
|
||||
exclude := ignore.CompileIgnoreLines("piper", ".pipeline")
|
||||
|
||||
projDescExists, err := utils.FileExists(config.ProjectDescriptor)
|
||||
if err != nil {
|
||||
log.SetErrorCategory(log.ErrorConfiguration)
|
||||
return errors.Wrap(err, "failed to check if project descriptor exists")
|
||||
}
|
||||
|
||||
if projDescExists {
|
||||
descriptor, err := project.ParseDescriptor(config.ProjectDescriptor, utils, httpClient)
|
||||
if err != nil {
|
||||
log.SetErrorCategory(log.ErrorConfiguration)
|
||||
return errors.Wrapf(err, "failed to parse %s", config.ProjectDescriptor)
|
||||
}
|
||||
|
||||
if (config.Buildpacks == nil || len(config.Buildpacks) == 0) && len(descriptor.Buildpacks) > 0 {
|
||||
config.Buildpacks = descriptor.Buildpacks
|
||||
}
|
||||
|
||||
if (config.BuildEnvVars == nil || len(config.BuildEnvVars) == 0) && len(descriptor.EnvVars) > 0 {
|
||||
config.BuildEnvVars = descriptor.EnvVars
|
||||
}
|
||||
|
||||
if descriptor.Exclude != nil {
|
||||
exclude = descriptor.Exclude
|
||||
}
|
||||
|
||||
if descriptor.Include != nil {
|
||||
include = descriptor.Include
|
||||
}
|
||||
}
|
||||
|
||||
platformPath := "/platform"
|
||||
if config.BuildEnvVars != nil && len(config.BuildEnvVars) > 0 {
|
||||
log.Entry().Infof("Setting custom environment variables: '%v'", config.BuildEnvVars)
|
||||
@ -250,13 +324,13 @@ func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, u
|
||||
}
|
||||
|
||||
if dir {
|
||||
err = copyProject(source, target, utils)
|
||||
err = copyProject(source, target, include, exclude, utils)
|
||||
if err != nil {
|
||||
log.SetErrorCategory(log.ErrorBuild)
|
||||
return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target)
|
||||
}
|
||||
} else {
|
||||
err = copyFile(source, target, utils)
|
||||
err = extractZip(source, target, utils)
|
||||
if err != nil {
|
||||
log.SetErrorCategory(log.ErrorBuild)
|
||||
return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target)
|
||||
|
||||
@ -24,6 +24,7 @@ type cnbBuildOptions struct {
|
||||
Buildpacks []string `json:"buildpacks,omitempty"`
|
||||
BuildEnvVars []string `json:"buildEnvVars,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
ProjectDescriptor string `json:"projectDescriptor,omitempty"`
|
||||
DockerConfigJSON string `json:"dockerConfigJSON,omitempty"`
|
||||
CustomTLSCertificateLinks []string `json:"customTlsCertificateLinks,omitempty"`
|
||||
AdditionalTags []string `json:"additionalTags,omitempty"`
|
||||
@ -152,6 +153,7 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) {
|
||||
cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of '<hostname>/<repo>[:<tag>]'.")
|
||||
cmd.Flags().StringSliceVar(&stepConfig.BuildEnvVars, "buildEnvVars", []string{}, "List of custom environment variables used during a build in the form of 'KEY=VALUE'.")
|
||||
cmd.Flags().StringVar(&stepConfig.Path, "path", os.Getenv("PIPER_path"), "The path should either point to a directory with your sources or an artifact in zip format.")
|
||||
cmd.Flags().StringVar(&stepConfig.ProjectDescriptor, "projectDescriptor", `project.toml`, "Path to the project.toml file (see https://buildpacks.io/docs/reference/config/project-descriptor/ for the reference). Parameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file.")
|
||||
cmd.Flags().StringVar(&stepConfig.DockerConfigJSON, "dockerConfigJSON", os.Getenv("PIPER_dockerConfigJSON"), "Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).")
|
||||
cmd.Flags().StringSliceVar(&stepConfig.CustomTLSCertificateLinks, "customTlsCertificateLinks", []string{}, "List containing download links of custom TLS certificates. This is required to ensure trusted connections to registries with custom certificates.")
|
||||
cmd.Flags().StringSliceVar(&stepConfig.AdditionalTags, "additionalTags", []string{}, "List of tags which will be additionally pushed to the registry, e.g. \"latest\".")
|
||||
@ -240,6 +242,15 @@ func cnbBuildMetadata() config.StepData {
|
||||
Aliases: []config.Alias{},
|
||||
Default: os.Getenv("PIPER_path"),
|
||||
},
|
||||
{
|
||||
Name: "projectDescriptor",
|
||||
ResourceRef: []config.ResourceReference{},
|
||||
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
|
||||
Type: "string",
|
||||
Mandatory: false,
|
||||
Aliases: []config.Alias{},
|
||||
Default: `project.toml`,
|
||||
},
|
||||
{
|
||||
Name: "dockerConfigJSON",
|
||||
ResourceRef: []config.ResourceReference{
|
||||
|
||||
1
go.mod
1
go.mod
@ -47,6 +47,7 @@ require (
|
||||
github.com/pelletier/go-toml v1.9.3
|
||||
github.com/piper-validation/fortify-client-go v0.0.0-20210114140201-1261216783c6
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
|
||||
github.com/sirupsen/logrus v1.7.0
|
||||
github.com/smartystreets/assertions v1.0.0 // indirect
|
||||
github.com/spf13/cobra v1.0.0
|
||||
|
||||
2
go.sum
2
go.sum
@ -1387,6 +1387,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
|
||||
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
|
||||
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
|
||||
github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
|
||||
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
|
||||
|
||||
@ -26,6 +26,28 @@ func TestNpmProject(t *testing.T) {
|
||||
container.assertHasOutput(t, "failed to write image to the following tags: [test/not-found:0.0.1")
|
||||
}
|
||||
|
||||
func TestProjectDescriptor(t *testing.T) {
|
||||
t.Parallel()
|
||||
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
|
||||
Image: "paketobuildpacks/builder:full",
|
||||
User: "cnb",
|
||||
TestDir: []string{"testdata", "TestCnbIntegration", "project"},
|
||||
})
|
||||
|
||||
container.whenRunningPiperCommand("cnbBuild", "-v", "--containerImageName", "not-found", "--containerImageTag", "0.0.1", "--containerRegistryUrl", "test")
|
||||
|
||||
container.assertHasOutput(t, "running command: /cnb/lifecycle/detector")
|
||||
container.assertHasOutput(t, "/project/Dockerfile doesn't match include pattern, ignoring")
|
||||
container.assertHasOutput(t, "/project/srv/hello.js matches include pattern")
|
||||
container.assertHasOutput(t, "/project/srv/hello.js matches include pattern")
|
||||
container.assertHasOutput(t, "Downloading buildpack")
|
||||
container.assertHasOutput(t, "Setting custom environment variables: '[BP_NODE_VERSION=15.14.0]'")
|
||||
container.assertHasOutput(t, "Selected Node Engine version (using BP_NODE_VERSION): 15.14.0")
|
||||
container.assertHasOutput(t, "Paketo NPM Start Buildpack")
|
||||
container.assertHasOutput(t, "Saving test/not-found:0.0.1")
|
||||
container.assertHasOutput(t, "failed to write image to the following tags: [test/not-found:0.0.1")
|
||||
}
|
||||
|
||||
func TestZipPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{
|
||||
|
||||
21
integration/testdata/TestCnbIntegration/project/.gitignore
vendored
Normal file
21
integration/testdata/TestCnbIntegration/project/.gitignore
vendored
Normal file
@ -0,0 +1,21 @@
|
||||
_out
|
||||
*.db
|
||||
connection.properties
|
||||
default-*.json
|
||||
node_modules/
|
||||
package-lock.json
|
||||
target/
|
||||
.che/
|
||||
.gen/
|
||||
*_mta_build_tmp
|
||||
*.mtar
|
||||
mta_archives/
|
||||
Makefile_*.mta
|
||||
.DS_Store
|
||||
*.orig
|
||||
*.log
|
||||
.pipeline
|
||||
mtaBuild_errorDetails.json
|
||||
test-mta-js.mtar
|
||||
.nyc_output/
|
||||
s4hana_pipeline
|
||||
3
integration/testdata/TestCnbIntegration/project/Dockerfile
vendored
Normal file
3
integration/testdata/TestCnbIntegration/project/Dockerfile
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
FROM devxci/mbtci:1.1.1
|
||||
|
||||
COPY run-in-container.sh /test.sh
|
||||
0
integration/testdata/TestCnbIntegration/project/gen/srv/.gitkeep
vendored
Normal file
0
integration/testdata/TestCnbIntegration/project/gen/srv/.gitkeep
vendored
Normal file
16
integration/testdata/TestCnbIntegration/project/mta.yaml
vendored
Normal file
16
integration/testdata/TestCnbIntegration/project/mta.yaml
vendored
Normal file
@ -0,0 +1,16 @@
|
||||
_schema-version: '3.1'
|
||||
ID: test-mta-js
|
||||
version: 1.0.0
|
||||
|
||||
build-parameters:
|
||||
before-all:
|
||||
- builder: custom
|
||||
commands:
|
||||
- npm install
|
||||
- npm run-script ci-backend-unit-test
|
||||
- npm run-script ci-it-backend
|
||||
|
||||
modules:
|
||||
- name: test-mta-js-srv
|
||||
type: nodejs
|
||||
path: gen/srv
|
||||
22
integration/testdata/TestCnbIntegration/project/package.json
vendored
Normal file
22
integration/testdata/TestCnbIntegration/project/package.json
vendored
Normal file
@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "test-mta-js",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"jest": "^26.0.1",
|
||||
"jest-jenkins-reporter": "^1.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^4.2.0",
|
||||
"mocha": "^8.0.1",
|
||||
"mocha-junit-reporter": "^2.0.0",
|
||||
"nyc": "^15.1.0",
|
||||
"sinon": "^9.0.2",
|
||||
"sinon-chai": "^3.5.0",
|
||||
"ts-node": "^8.10.2",
|
||||
"typescript": "^3.9.5"
|
||||
},
|
||||
"scripts": {
|
||||
"ci-backend-unit-test": "nyc --report-dir ./s4hana_pipeline/reports/coverage-reports/backend-unit/ --reporter cobertura mocha -r ts-node/register test/unit-tests/*.spec.js --reporter mocha-junit-reporter --reporter-options mochaFile=./s4hana_pipeline/reports/backend-unit/results.xml",
|
||||
"ci-it-backend": "nyc --report-dir ./s4hana_pipeline/reports/coverage-reports/backend-integration/ --reporter cobertura mocha -r ts-node/register test/integration-tests/*.spec.js --reporter mocha-junit-reporter --reporter-options mochaFile=./s4hana_pipeline/reports/backend-integration/results.xml"
|
||||
}
|
||||
}
|
||||
31
integration/testdata/TestCnbIntegration/project/project.toml
vendored
Normal file
31
integration/testdata/TestCnbIntegration/project/project.toml
vendored
Normal file
@ -0,0 +1,31 @@
|
||||
[project]
|
||||
id = "io.buildpacks.my-app"
|
||||
version = "0.1"
|
||||
|
||||
[build]
|
||||
include = [
|
||||
"*.js",
|
||||
"package.json"
|
||||
]
|
||||
|
||||
[[build.env]]
|
||||
name = "BP_NODE_VERSION"
|
||||
value = "15.14.0"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "paketo-buildpacks/ca-certificates"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "paketo-buildpacks/node-engine"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "paketo-buildpacks/npm-install"
|
||||
version = "0.4.0"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "paketo-buildpacks/node-module-bom"
|
||||
version = "0.1.2"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "paketo-buildpacks/npm-start"
|
||||
version = "0.4.0"
|
||||
8
integration/testdata/TestCnbIntegration/project/srv/hello.js
vendored
Normal file
8
integration/testdata/TestCnbIntegration/project/srv/hello.js
vendored
Normal file
@ -0,0 +1,8 @@
|
||||
module.exports = {
|
||||
sum: function(a,b) {
|
||||
return a+b
|
||||
},
|
||||
multiply: function(a,b) {
|
||||
return a*b
|
||||
}
|
||||
};
|
||||
6
integration/testdata/TestCnbIntegration/project/test/integration-tests/helloworld.spec.js
vendored
Normal file
6
integration/testdata/TestCnbIntegration/project/test/integration-tests/helloworld.spec.js
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
|
||||
describe("hello IT", () => {
|
||||
it("responds with \"Hello, World!\"", () => {
|
||||
|
||||
});
|
||||
});
|
||||
9
integration/testdata/TestCnbIntegration/project/test/unit-tests/helloworld.spec.js
vendored
Normal file
9
integration/testdata/TestCnbIntegration/project/test/unit-tests/helloworld.spec.js
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
const hello = require('../../srv/hello.js')
|
||||
|
||||
describe("hello world route", () => {
|
||||
it("responds with \"Hello, World!\"", () => {
|
||||
|
||||
hello.sum(1,2)
|
||||
hello.multiply(1,1)
|
||||
});
|
||||
});
|
||||
@ -3,7 +3,7 @@ package cnbutils
|
||||
import (
|
||||
"bytes"
|
||||
|
||||
"github.com/pelletier/go-toml"
|
||||
toml "github.com/pelletier/go-toml"
|
||||
)
|
||||
|
||||
type Order struct {
|
||||
|
||||
128
pkg/cnbutils/project/descriptor.go
Normal file
128
pkg/cnbutils/project/descriptor.go
Normal file
@ -0,0 +1,128 @@
|
||||
// Package project handles project.toml parsing
|
||||
package project
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/SAP/jenkins-library/pkg/cnbutils"
|
||||
"github.com/SAP/jenkins-library/pkg/cnbutils/registry"
|
||||
piperhttp "github.com/SAP/jenkins-library/pkg/http"
|
||||
"github.com/SAP/jenkins-library/pkg/log"
|
||||
toml "github.com/pelletier/go-toml"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
)
|
||||
|
||||
type script struct {
|
||||
API string `toml:"api"`
|
||||
Inline string `toml:"inline"`
|
||||
Shell string `toml:"shell"`
|
||||
}
|
||||
type buildpack struct {
|
||||
ID string `toml:"id"`
|
||||
Version string `toml:"version"`
|
||||
URI string `toml:"uri"`
|
||||
Script script `toml:"script"`
|
||||
}
|
||||
|
||||
type envVar struct {
|
||||
Name string `toml:"name"`
|
||||
Value string `toml:"value"`
|
||||
}
|
||||
|
||||
type build struct {
|
||||
Include []string `toml:"include"`
|
||||
Exclude []string `toml:"exclude"`
|
||||
Buildpacks []buildpack `toml:"buildpacks"`
|
||||
Env []envVar `toml:"env"`
|
||||
}
|
||||
|
||||
type projectDescriptor struct {
|
||||
Build build `toml:"build"`
|
||||
Metadata map[string]interface{} `toml:"metadata"`
|
||||
}
|
||||
|
||||
type Descriptor struct {
|
||||
Exclude *ignore.GitIgnore
|
||||
Include *ignore.GitIgnore
|
||||
EnvVars []string
|
||||
Buildpacks []string
|
||||
}
|
||||
|
||||
func ParseDescriptor(descriptorPath string, utils cnbutils.BuildUtils, httpClient piperhttp.Sender) (Descriptor, error) {
|
||||
descriptor := Descriptor{}
|
||||
|
||||
descriptorContent, err := utils.FileRead(descriptorPath)
|
||||
if err != nil {
|
||||
return Descriptor{}, err
|
||||
}
|
||||
|
||||
rawDescriptor := projectDescriptor{}
|
||||
err = toml.Unmarshal(descriptorContent, &rawDescriptor)
|
||||
if err != nil {
|
||||
return Descriptor{}, err
|
||||
}
|
||||
|
||||
if rawDescriptor.Build.Buildpacks != nil && len(rawDescriptor.Build.Buildpacks) > 0 {
|
||||
buildpacksImg, err := rawDescriptor.Build.searchBuildpacks(httpClient)
|
||||
if err != nil {
|
||||
return Descriptor{}, err
|
||||
}
|
||||
|
||||
descriptor.Buildpacks = buildpacksImg
|
||||
}
|
||||
|
||||
if rawDescriptor.Build.Env != nil && len(rawDescriptor.Build.Env) > 0 {
|
||||
descriptor.EnvVars = rawDescriptor.Build.envToStringSlice()
|
||||
}
|
||||
|
||||
if rawDescriptor.Build.Exclude != nil && len(rawDescriptor.Build.Exclude) > 0 {
|
||||
descriptor.Exclude = ignore.CompileIgnoreLines(rawDescriptor.Build.Exclude...)
|
||||
}
|
||||
|
||||
if rawDescriptor.Build.Include != nil && len(rawDescriptor.Build.Include) > 0 {
|
||||
descriptor.Include = ignore.CompileIgnoreLines(rawDescriptor.Build.Include...)
|
||||
}
|
||||
|
||||
return descriptor, nil
|
||||
}
|
||||
|
||||
func (b *build) envToStringSlice() []string {
|
||||
strSlice := []string{}
|
||||
|
||||
for _, e := range b.Env {
|
||||
if len(e.Name) == 0 || len(e.Value) == 0 {
|
||||
continue
|
||||
}
|
||||
strSlice = append(strSlice, fmt.Sprintf("%s=%s", e.Name, e.Value))
|
||||
}
|
||||
|
||||
return strSlice
|
||||
}
|
||||
|
||||
func (b *build) searchBuildpacks(httpClient piperhttp.Sender) ([]string, error) {
|
||||
var bpackImg []string
|
||||
|
||||
for _, bpack := range b.Buildpacks {
|
||||
if bpack.Script != (script{}) {
|
||||
return nil, errors.New("inline buildpacks are not supported")
|
||||
}
|
||||
|
||||
if bpack.URI != "" {
|
||||
log.Entry().Debugf("Adding buildpack using URI: %s", bpack.URI)
|
||||
bpackImg = append(bpackImg, bpack.URI)
|
||||
} else if bpack.ID != "" {
|
||||
imgURL, err := registry.SearchBuildpack(bpack.ID, bpack.Version, httpClient, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bpackImg = append(bpackImg, imgURL)
|
||||
} else {
|
||||
return nil, errors.New("invalid buildpack entry in project.toml, either URI or ID should be specified")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return bpackImg, nil
|
||||
}
|
||||
139
pkg/cnbutils/project/descriptor_test.go
Normal file
139
pkg/cnbutils/project/descriptor_test.go
Normal file
@ -0,0 +1,139 @@
|
||||
package project
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/SAP/jenkins-library/pkg/cnbutils"
|
||||
"github.com/SAP/jenkins-library/pkg/mock"
|
||||
"github.com/jarcoal/httpmock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
piperhttp "github.com/SAP/jenkins-library/pkg/http"
|
||||
)
|
||||
|
||||
func TestParseDescriptor(t *testing.T) {
|
||||
t.Run("parses the project.toml file", func(t *testing.T) {
|
||||
projectToml := `[project]
|
||||
id = "io.buildpacks.my-app"
|
||||
version = "0.1"
|
||||
|
||||
[build]
|
||||
include = [
|
||||
"cmd/",
|
||||
"go.mod",
|
||||
"go.sum",
|
||||
"*.go"
|
||||
]
|
||||
exclude = [
|
||||
".pipeline"
|
||||
]
|
||||
|
||||
[[build.env]]
|
||||
name = "VAR1"
|
||||
value = "VAL1"
|
||||
|
||||
[[build.env]]
|
||||
name = "VAR2"
|
||||
value = "VAL2"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "paketo-buildpacks/java"
|
||||
version = "5.9.1"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "paketo-buildpacks/nodejs"
|
||||
`
|
||||
utils := cnbutils.MockUtils{
|
||||
FilesMock: &mock.FilesMock{},
|
||||
}
|
||||
|
||||
fakeJavaResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"5.9.1\",\"_link\":\"https://test-java/5.9.1\"}]}"
|
||||
fakeNodeJsResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"1.1.1\",\"_link\":\"https://test-nodejs/1.1.1\"}]}"
|
||||
|
||||
utils.AddFile("project.toml", []byte(projectToml))
|
||||
httpmock.Activate()
|
||||
defer httpmock.DeactivateAndReset()
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/paketo-buildpacks/java", httpmock.NewStringResponder(200, fakeJavaResponse))
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/paketo-buildpacks/nodejs", httpmock.NewStringResponder(200, fakeNodeJsResponse))
|
||||
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://test-java/5.9.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test-java@5.9.1\"}"))
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://test-nodejs/1.1.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test-nodejs@1.1.1\"}"))
|
||||
client := &piperhttp.Client{}
|
||||
client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
|
||||
|
||||
descriptor, err := ParseDescriptor("project.toml", utils, client)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, descriptor.EnvVars, "VAR1=VAL1")
|
||||
assert.Contains(t, descriptor.EnvVars, "VAR2=VAL2")
|
||||
|
||||
assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-java@5.9.1")
|
||||
assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-nodejs@1.1.1")
|
||||
|
||||
assert.NotNil(t, descriptor.Exclude)
|
||||
assert.NotNil(t, descriptor.Include)
|
||||
|
||||
t1 := descriptor.Exclude.MatchesPath(".pipeline/commonEnv.yaml")
|
||||
assert.True(t, t1)
|
||||
|
||||
t2 := descriptor.Exclude.MatchesPath("src/java/test.java")
|
||||
assert.False(t, t2)
|
||||
|
||||
t3 := descriptor.Include.MatchesPath("cmd/cobra.go")
|
||||
assert.True(t, t3)
|
||||
|
||||
t4 := descriptor.Include.MatchesPath("pkg/test/main.go")
|
||||
assert.True(t, t4)
|
||||
|
||||
t5 := descriptor.Include.MatchesPath("Makefile")
|
||||
assert.False(t, t5)
|
||||
})
|
||||
|
||||
t.Run("fails with inline buildpack", func(t *testing.T) {
|
||||
projectToml := `[project]
|
||||
id = "io.buildpacks.my-app"
|
||||
version = "0.1"
|
||||
|
||||
[[build.buildpacks]]
|
||||
id = "test/inline"
|
||||
[build.buildpacks.script]
|
||||
api = "0.5"
|
||||
shell = "/bin/bash"
|
||||
inline = "date"
|
||||
`
|
||||
utils := cnbutils.MockUtils{
|
||||
FilesMock: &mock.FilesMock{},
|
||||
}
|
||||
|
||||
utils.AddFile("project.toml", []byte(projectToml))
|
||||
|
||||
_, err := ParseDescriptor("project.toml", utils, &piperhttp.Client{})
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "inline buildpacks are not supported", err.Error())
|
||||
})
|
||||
|
||||
t.Run("fails with file not found", func(t *testing.T) {
|
||||
utils := cnbutils.MockUtils{
|
||||
FilesMock: &mock.FilesMock{},
|
||||
}
|
||||
|
||||
_, err := ParseDescriptor("project.toml", utils, &piperhttp.Client{})
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "could not read 'project.toml'", err.Error())
|
||||
})
|
||||
|
||||
t.Run("fails to parse corrupted project.toml", func(t *testing.T) {
|
||||
projectToml := "test123"
|
||||
utils := cnbutils.MockUtils{
|
||||
FilesMock: &mock.FilesMock{},
|
||||
}
|
||||
utils.AddFile("project.toml", []byte(projectToml))
|
||||
_, err := ParseDescriptor("project.toml", utils, &piperhttp.Client{})
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "(1, 8): was expecting token =, but got EOF instead", err.Error())
|
||||
})
|
||||
}
|
||||
86
pkg/cnbutils/registry/search.go
Normal file
86
pkg/cnbutils/registry/search.go
Normal file
@ -0,0 +1,86 @@
|
||||
// Package registry provides utilities to search buildpacks using registry API
|
||||
package registry
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
piperhttp "github.com/SAP/jenkins-library/pkg/http"
|
||||
"github.com/SAP/jenkins-library/pkg/log"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultRegistryAPI = "https://registry.buildpacks.io/api/v1/buildpacks"
|
||||
)
|
||||
|
||||
type latest struct {
|
||||
Version string `json:"version"`
|
||||
Misc map[string]interface{} `json:"-"`
|
||||
}
|
||||
|
||||
type version struct {
|
||||
Version string `json:"version"`
|
||||
Link string `json:"_link"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Latest latest `json:"latest"`
|
||||
Versions []version `json:"versions"`
|
||||
}
|
||||
|
||||
type versionResponse struct {
|
||||
Addr string `json:"addr"`
|
||||
Misc map[string]interface{} `json:"-"`
|
||||
}
|
||||
|
||||
func SearchBuildpack(id, version string, httpClient piperhttp.Sender, baseApiURL string) (string, error) {
|
||||
var apiResponse response
|
||||
|
||||
if baseApiURL == "" {
|
||||
baseApiURL = defaultRegistryAPI
|
||||
}
|
||||
|
||||
apiURL := fmt.Sprintf("%s/%s", baseApiURL, id)
|
||||
|
||||
rawResponse, err := httpClient.SendRequest(http.MethodGet, apiURL, nil, nil, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rawResponse.Body.Close()
|
||||
|
||||
err = json.NewDecoder(rawResponse.Body).Decode(&apiResponse)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to parse response from the %s, error: %s", apiURL, err.Error())
|
||||
}
|
||||
|
||||
if version == "" {
|
||||
version = apiResponse.Latest.Version
|
||||
log.Entry().Infof("Version for the buildpack '%s' is not specified, using the latest '%s'", id, version)
|
||||
}
|
||||
|
||||
for _, ver := range apiResponse.Versions {
|
||||
if ver.Version == version {
|
||||
return getImageAddr(ver.Link, httpClient)
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("version '%s' was not found for the buildpack '%s'", version, id)
|
||||
}
|
||||
|
||||
func getImageAddr(link string, httpClient piperhttp.Sender) (string, error) {
|
||||
var verResponse versionResponse
|
||||
|
||||
rawResponse, err := httpClient.SendRequest(http.MethodGet, link, nil, nil, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rawResponse.Body.Close()
|
||||
|
||||
err = json.NewDecoder(rawResponse.Body).Decode(&verResponse)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to parse response from the %s, error: %s", link, err.Error())
|
||||
}
|
||||
|
||||
return verResponse.Addr, nil
|
||||
}
|
||||
90
pkg/cnbutils/registry/search_test.go
Normal file
90
pkg/cnbutils/registry/search_test.go
Normal file
@ -0,0 +1,90 @@
|
||||
package registry
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
piperhttp "github.com/SAP/jenkins-library/pkg/http"
|
||||
"github.com/jarcoal/httpmock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSearchBuildpack(t *testing.T) {
|
||||
t.Run("returns image URL for specific version", func(t *testing.T) {
|
||||
|
||||
fakeResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"1.1.1\",\"_link\":\"https://test/1.1.1\"}]}"
|
||||
|
||||
httpmock.Activate()
|
||||
defer httpmock.DeactivateAndReset()
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/test", httpmock.NewStringResponder(200, fakeResponse))
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://test/1.1.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test@1.1.1\"}"))
|
||||
client := &piperhttp.Client{}
|
||||
client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
|
||||
|
||||
img, err := SearchBuildpack("test", "1.1.1", client, "")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "index.docker.io/test@1.1.1", img)
|
||||
})
|
||||
|
||||
t.Run("returns image URL for the latest", func(t *testing.T) {
|
||||
fakeResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"1.1.1\",\"_link\":\"https://test/1.1.1\"}]}"
|
||||
|
||||
httpmock.Activate()
|
||||
defer httpmock.DeactivateAndReset()
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/test", httpmock.NewStringResponder(200, fakeResponse))
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://test/1.1.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test@1.1.1\"}"))
|
||||
client := &piperhttp.Client{}
|
||||
client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
|
||||
|
||||
img, err := SearchBuildpack("test", "", client, "")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "index.docker.io/test@1.1.1", img)
|
||||
})
|
||||
|
||||
t.Run("fails with the version not found", func(t *testing.T) {
|
||||
fakeResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"1.1.1\",\"_link\":\"https://test/1.1.1\"}]}"
|
||||
|
||||
httpmock.Activate()
|
||||
defer httpmock.DeactivateAndReset()
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/test", httpmock.NewStringResponder(200, fakeResponse))
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://test/1.1.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test@1.1.1\"}"))
|
||||
client := &piperhttp.Client{}
|
||||
client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
|
||||
|
||||
img, err := SearchBuildpack("test", "1.1.2", client, "")
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, "version '1.1.2' was not found for the buildpack 'test'", err.Error())
|
||||
assert.Equal(t, "", img)
|
||||
})
|
||||
|
||||
t.Run("fails with the HTTP error", func(t *testing.T) {
|
||||
httpmock.Activate()
|
||||
defer httpmock.DeactivateAndReset()
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/test", httpmock.NewStringResponder(404, "not_found"))
|
||||
client := &piperhttp.Client{}
|
||||
client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
|
||||
|
||||
img, err := SearchBuildpack("test", "1.1.2", client, "")
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "returned with response 404")
|
||||
assert.Equal(t, "", img)
|
||||
})
|
||||
|
||||
t.Run("fails with the invalid response object", func(t *testing.T) {
|
||||
httpmock.Activate()
|
||||
defer httpmock.DeactivateAndReset()
|
||||
httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/test", httpmock.NewStringResponder(200, "not_a_json"))
|
||||
client := &piperhttp.Client{}
|
||||
client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true})
|
||||
|
||||
img, err := SearchBuildpack("test", "1.1.2", client, "")
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unable to parse response from the https://registry.buildpacks.io/api/v1/buildpacks/test, error: invalid character")
|
||||
assert.Equal(t, "", img)
|
||||
})
|
||||
}
|
||||
@ -15,6 +15,7 @@ import (
|
||||
// FileUtils ...
|
||||
type FileUtils interface {
|
||||
Abs(path string) (string, error)
|
||||
DirExists(path string) (bool, error)
|
||||
FileExists(filename string) (bool, error)
|
||||
Copy(src, dest string) (int64, error)
|
||||
FileRead(path string) ([]byte, error)
|
||||
|
||||
@ -92,6 +92,14 @@ spec:
|
||||
- PARAMETERS
|
||||
- STAGES
|
||||
- STEPS
|
||||
- name: projectDescriptor
|
||||
type: string
|
||||
description: Path to the project.toml file (see https://buildpacks.io/docs/reference/config/project-descriptor/ for the reference). Parameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file.
|
||||
default: project.toml
|
||||
scope:
|
||||
- PARAMETERS
|
||||
- STAGES
|
||||
- STEPS
|
||||
- name: dockerConfigJSON
|
||||
type: string
|
||||
description: Path to the file `.docker/config.json` - this is typically provided by your CI/CD system. You can find more details about the Docker credentials in the [Docker documentation](https://docs.docker.com/engine/reference/commandline/login/).
|
||||
|
||||
Loading…
Reference in New Issue
Block a user