1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2025-01-18 05:18:24 +02:00

Implement npmExecuteScripts step (#1422)

This commit is contained in:
Florian Wilhelm 2020-04-24 18:29:30 +02:00 committed by GitHub
parent fb4cfd84ec
commit 3f5b9cc555
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 596 additions and 1 deletions

203
cmd/npmExecuteScripts.go Normal file
View File

@ -0,0 +1,203 @@
package cmd
import (
"bytes"
"github.com/SAP/jenkins-library/pkg/command"
"github.com/SAP/jenkins-library/pkg/log"
FileUtils "github.com/SAP/jenkins-library/pkg/piperutils"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/bmatcuk/doublestar"
"os"
"path"
"strings"
)
type npmExecuteScriptsUtilsInterface interface {
fileExists(path string) (bool, error)
glob(pattern string) (matches []string, err error)
getwd() (dir string, err error)
chdir(dir string) error
getExecRunner() execRunner
}
type npmExecuteScriptsUtilsBundle struct {
projectStructure FileUtils.ProjectStructure
fileUtils FileUtils.Files
execRunner *command.Command
}
func (u *npmExecuteScriptsUtilsBundle) fileExists(path string) (bool, error) {
return u.fileUtils.FileExists(path)
}
func (u *npmExecuteScriptsUtilsBundle) glob(pattern string) (matches []string, err error) {
return doublestar.Glob(pattern)
}
func (u *npmExecuteScriptsUtilsBundle) getwd() (dir string, err error) {
return os.Getwd()
}
func (u *npmExecuteScriptsUtilsBundle) chdir(dir string) error {
return os.Chdir(dir)
}
func (u *npmExecuteScriptsUtilsBundle) getExecRunner() execRunner {
if u.execRunner == nil {
u.execRunner = &command.Command{}
u.execRunner.Stdout(log.Entry().Writer())
u.execRunner.Stderr(log.Entry().Writer())
}
return u.execRunner
}
func npmExecuteScripts(config npmExecuteScriptsOptions, telemetryData *telemetry.CustomData) {
utils := npmExecuteScriptsUtilsBundle{}
err := runNpmExecuteScripts(&utils, &config)
if err != nil {
log.Entry().WithError(err).Fatal("step execution failed")
}
}
func runNpmExecuteScripts(utils npmExecuteScriptsUtilsInterface, options *npmExecuteScriptsOptions) error {
execRunner := utils.getExecRunner()
packageJSONFiles, err := findPackageJSONFiles(utils)
if err != nil {
return err
}
oldWorkingDirectory, err := utils.getwd()
if err != nil {
return err
}
for _, file := range packageJSONFiles {
dir := path.Dir(file)
err = utils.chdir(dir)
if err != nil {
return err
}
// set in each directory to respect existing config in rc files
err = setNpmRegistries(options, execRunner)
if err != nil {
return err
}
packageLockExists, yarnLockExists, err := checkIfLockFilesExist(utils)
if err != nil {
return err
}
if options.Install {
err = installDependencies(dir, packageLockExists, yarnLockExists, execRunner)
if err != nil {
return err
}
}
for _, v := range options.RunScripts {
log.Entry().WithField("WorkingDirectory", dir).Info("run-script " + v)
err = execRunner.RunExecutable("npm", "run-script", v, "--if-present")
if err != nil {
return err
}
}
err = utils.chdir(oldWorkingDirectory)
if err != nil {
return err
}
}
return err
}
func setNpmRegistries(options *npmExecuteScriptsOptions, execRunner execRunner) error {
environment := []string{}
const sapRegistry = "@sap:registry"
const npmRegistry = "registry"
configurableRegistries := []string{npmRegistry, sapRegistry}
for _, registry := range configurableRegistries {
var buffer bytes.Buffer
execRunner.Stdout(&buffer)
err := execRunner.RunExecutable("npm", "config", "get", registry)
execRunner.Stdout(log.Entry().Writer())
if err != nil {
return err
}
preConfiguredRegistry := buffer.String()
log.Entry().Info("Discovered pre-configured npm registry " + preConfiguredRegistry)
if registry == npmRegistry && options.DefaultNpmRegistry != "" && (preConfiguredRegistry == "undefined" || strings.HasPrefix(preConfiguredRegistry, "https://registry.npmjs.org")) {
log.Entry().Info("npm registry " + registry + " was not configured, setting it to " + options.DefaultNpmRegistry)
environment = append(environment, "npm_config_"+registry+"="+options.DefaultNpmRegistry)
}
if registry == sapRegistry && (preConfiguredRegistry == "undefined" || strings.HasPrefix(preConfiguredRegistry, "https://npm.sap.com")) {
log.Entry().Info("npm registry " + registry + " was not configured, setting it to " + options.SapNpmRegistry)
environment = append(environment, "npm_config_"+registry+"="+options.SapNpmRegistry)
}
}
log.Entry().Info("Setting environment: " + strings.Join(environment, ", "))
execRunner.SetEnv(environment)
return nil
}
func findPackageJSONFiles(utils npmExecuteScriptsUtilsInterface) ([]string, error) {
unfilteredListOfPackageJSONFiles, err := utils.glob("**/package.json")
if err != nil {
return nil, err
}
var packageJSONFiles []string
for _, file := range unfilteredListOfPackageJSONFiles {
if strings.Contains(file, "node_modules") {
continue
}
if strings.HasPrefix(file, "gen/") || strings.Contains(file, "/gen/") {
continue
}
packageJSONFiles = append(packageJSONFiles, file)
log.Entry().Info("Discovered package.json file " + file)
}
return packageJSONFiles, nil
}
func checkIfLockFilesExist(utils npmExecuteScriptsUtilsInterface) (bool, bool, error) {
packageLockExists, err := utils.fileExists("package-lock.json")
if err != nil {
return false, false, err
}
yarnLockExists, err := utils.fileExists("yarn.lock")
if err != nil {
return false, false, err
}
return packageLockExists, yarnLockExists, nil
}
func installDependencies(dir string, packageLockExists bool, yarnLockExists bool, execRunner execRunner) (err error) {
log.Entry().WithField("WorkingDirectory", dir).Info("Running install")
if packageLockExists {
err = execRunner.RunExecutable("npm", "ci")
if err != nil {
return err
}
} else if yarnLockExists {
err = execRunner.RunExecutable("yarn", "install", "--frozen-lockfile")
if err != nil {
return err
}
} else {
log.Entry().Warn("No package lock file found. " +
"It is recommended to create a `package-lock.json` file by running `npm install` locally." +
" Add this file to your version control. " +
"By doing so, the builds of your application become more reliable.")
err = execRunner.RunExecutable("npm", "install")
if err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,117 @@
// Code generated by piper's step-generator. DO NOT EDIT.
package cmd
import (
"fmt"
"os"
"time"
"github.com/SAP/jenkins-library/pkg/config"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/SAP/jenkins-library/pkg/telemetry"
"github.com/spf13/cobra"
)
type npmExecuteScriptsOptions struct {
Install bool `json:"install,omitempty"`
RunScripts []string `json:"runScripts,omitempty"`
DefaultNpmRegistry string `json:"defaultNpmRegistry,omitempty"`
SapNpmRegistry string `json:"sapNpmRegistry,omitempty"`
}
// NpmExecuteScriptsCommand Execute npm run scripts on all npm packages in a project
func NpmExecuteScriptsCommand() *cobra.Command {
metadata := npmExecuteScriptsMetadata()
var stepConfig npmExecuteScriptsOptions
var startTime time.Time
var createNpmExecuteScriptsCmd = &cobra.Command{
Use: "npmExecuteScripts",
Short: "Execute npm run scripts on all npm packages in a project",
Long: `Execute npm run scripts in all package json files, if they implement the scripts.`,
PreRunE: func(cmd *cobra.Command, args []string) error {
startTime = time.Now()
log.SetStepName("npmExecuteScripts")
log.SetVerbose(GeneralConfig.Verbose)
err := PrepareConfig(cmd, &metadata, "npmExecuteScripts", &stepConfig, config.OpenPiperFile)
if err != nil {
return err
}
return nil
},
Run: func(cmd *cobra.Command, args []string) {
telemetryData := telemetry.CustomData{}
telemetryData.ErrorCode = "1"
handler := func() {
telemetryData.Duration = fmt.Sprintf("%v", time.Since(startTime).Milliseconds())
telemetry.Send(&telemetryData)
}
log.DeferExitHandler(handler)
defer handler()
telemetry.Initialize(GeneralConfig.NoTelemetry, "npmExecuteScripts")
npmExecuteScripts(stepConfig, &telemetryData)
telemetryData.ErrorCode = "0"
},
}
addNpmExecuteScriptsFlags(createNpmExecuteScriptsCmd, &stepConfig)
return createNpmExecuteScriptsCmd
}
func addNpmExecuteScriptsFlags(cmd *cobra.Command, stepConfig *npmExecuteScriptsOptions) {
cmd.Flags().BoolVar(&stepConfig.Install, "install", false, "Run npm install or similar commands depending on the project structure.")
cmd.Flags().StringSliceVar(&stepConfig.RunScripts, "runScripts", []string{}, "List of additional run scripts to execute from package.json.")
cmd.Flags().StringVar(&stepConfig.DefaultNpmRegistry, "defaultNpmRegistry", os.Getenv("PIPER_defaultNpmRegistry"), "URL of the npm registry to use. Defaults to https://registry.npmjs.org/")
cmd.Flags().StringVar(&stepConfig.SapNpmRegistry, "sapNpmRegistry", "https://npm.sap.com", "The default npm registry URL to be used as the remote mirror for the SAP npm packages.")
}
// retrieve step metadata
func npmExecuteScriptsMetadata() config.StepData {
var theMetaData = config.StepData{
Metadata: config.StepMetadata{
Name: "npmExecuteScripts",
Aliases: []config.Alias{{Name: "executeNpm", Deprecated: false}},
},
Spec: config.StepSpec{
Inputs: config.StepInputs{
Parameters: []config.StepParameters{
{
Name: "install",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "bool",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "runScripts",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "[]string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "defaultNpmRegistry",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
{
Name: "sapNpmRegistry",
ResourceRef: []config.ResourceReference{},
Scope: []string{"PARAMETERS", "STAGES", "STEPS"},
Type: "string",
Mandatory: false,
Aliases: []config.Alias{},
},
},
},
},
}
return theMetaData
}

View File

@ -0,0 +1,16 @@
package cmd
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNpmExecuteScriptsCommand(t *testing.T) {
testCmd := NpmExecuteScriptsCommand()
// only high level testing performed - details are tested in step generation procudure
assert.Equal(t, "npmExecuteScripts", testCmd.Use, "command name incorrect")
}

View File

@ -0,0 +1,138 @@
package cmd
import (
"github.com/SAP/jenkins-library/pkg/mock"
"github.com/bmatcuk/doublestar"
"github.com/stretchr/testify/assert"
"sort"
"testing"
)
type npmExecuteScriptsMockUtilsBundle struct {
execRunner mock.ExecMockRunner
files map[string][]byte
}
func (u *npmExecuteScriptsMockUtilsBundle) fileExists(path string) (bool, error) {
_, exists := u.files[path]
return exists, nil
}
// duplicated from nexusUpload_test.go for now, refactor later?
func (u *npmExecuteScriptsMockUtilsBundle) glob(pattern string) ([]string, error) {
var matches []string
for path := range u.files {
matched, _ := doublestar.Match(pattern, path)
if matched {
matches = append(matches, path)
}
}
// The order in m.files is not deterministic, this would result in flaky tests.
sort.Sort(byLen(matches))
return matches, nil
}
func (u *npmExecuteScriptsMockUtilsBundle) getwd() (dir string, err error) {
return "/project", nil
}
func (u *npmExecuteScriptsMockUtilsBundle) chdir(dir string) error {
return nil
}
func (u *npmExecuteScriptsMockUtilsBundle) getExecRunner() execRunner {
return &u.execRunner
}
func TestNpmExecuteScripts(t *testing.T) {
t.Run("Call without install and run-scripts", func(t *testing.T) {
utils := newNpmExecuteScriptsMockUtilsBundle()
utils.files["package.json"] = []byte(`abc`)
utils.files["package-lock.json"] = []byte(`abc`)
options := npmExecuteScriptsOptions{}
err := runNpmExecuteScripts(&utils, &options)
assert.NoError(t, err)
assert.Equal(t, 2, len(utils.execRunner.Calls))
})
t.Run("Project with package lock", func(t *testing.T) {
utils := newNpmExecuteScriptsMockUtilsBundle()
utils.files["package.json"] = []byte(`abc`)
utils.files["foo/bar/node_modules/package.json"] = []byte(`abc`) // is filtered out
utils.files["gen/bar/package.json"] = []byte(`abc`) // is filtered out
utils.files["foo/gen/package.json"] = []byte(`abc`) // is filtered out
utils.files["package-lock.json"] = []byte(`abc`)
options := npmExecuteScriptsOptions{}
options.Install = true
options.RunScripts = []string{"foo", "bar"}
options.DefaultNpmRegistry = "foo.bar"
err := runNpmExecuteScripts(&utils, &options)
assert.NoError(t, err)
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"ci"}}, utils.execRunner.Calls[2])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "foo", "--if-present"}}, utils.execRunner.Calls[3])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "bar", "--if-present"}}, utils.execRunner.Calls[4])
assert.Equal(t, 5, len(utils.execRunner.Calls))
})
t.Run("Project with two package json files", func(t *testing.T) {
utils := newNpmExecuteScriptsMockUtilsBundle()
utils.files["package.json"] = []byte(`abc`)
utils.files["foo/bar/package.json"] = []byte(`abc`)
utils.files["package-lock.json"] = []byte(`abc`)
options := npmExecuteScriptsOptions{}
options.Install = true
options.RunScripts = []string{"foo", "bar"}
err := runNpmExecuteScripts(&utils, &options)
assert.NoError(t, err)
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"ci"}}, utils.execRunner.Calls[2])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "foo", "--if-present"}}, utils.execRunner.Calls[3])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "bar", "--if-present"}}, utils.execRunner.Calls[4])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"ci"}}, utils.execRunner.Calls[7])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "foo", "--if-present"}}, utils.execRunner.Calls[8])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "bar", "--if-present"}}, utils.execRunner.Calls[9])
assert.Equal(t, 10, len(utils.execRunner.Calls))
})
t.Run("Project with yarn lock", func(t *testing.T) {
utils := newNpmExecuteScriptsMockUtilsBundle()
utils.files["package.json"] = []byte(`abc`)
utils.files["yarn.lock"] = []byte(`abc`)
options := npmExecuteScriptsOptions{}
options.Install = true
options.RunScripts = []string{"foo", "bar"}
err := runNpmExecuteScripts(&utils, &options)
assert.NoError(t, err)
assert.Equal(t, mock.ExecCall{Exec: "yarn", Params: []string{"install", "--frozen-lockfile"}}, utils.execRunner.Calls[2])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "foo", "--if-present"}}, utils.execRunner.Calls[3])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "bar", "--if-present"}}, utils.execRunner.Calls[4])
})
t.Run("Project without lock file", func(t *testing.T) {
utils := newNpmExecuteScriptsMockUtilsBundle()
utils.files["package.json"] = []byte(`abc`)
options := npmExecuteScriptsOptions{}
options.Install = true
options.RunScripts = []string{"foo", "bar"}
err := runNpmExecuteScripts(&utils, &options)
assert.NoError(t, err)
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"install"}}, utils.execRunner.Calls[2])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "foo", "--if-present"}}, utils.execRunner.Calls[3])
assert.Equal(t, mock.ExecCall{Exec: "npm", Params: []string{"run-script", "bar", "--if-present"}}, utils.execRunner.Calls[4])
})
}
func newNpmExecuteScriptsMockUtilsBundle() npmExecuteScriptsMockUtilsBundle {
utils := npmExecuteScriptsMockUtilsBundle{}
utils.files = map[string][]byte{}
return utils
}

View File

@ -34,7 +34,7 @@ var rootCmd = &cobra.Command{
Use: "piper",
Short: "Executes CI/CD steps from project 'Piper' ",
Long: `
This project 'Piper' binary provides a CI/CD step libary.
This project 'Piper' binary provides a CI/CD step library.
It contains many steps which can be used within CI/CD systems as well as directly on e.g. a developer's machine.
`,
//ToDo: respect stageName to also come from parametersJSON -> first env.STAGE_NAME, second: parametersJSON, third: flag
@ -66,6 +66,7 @@ func Execute() {
rootCmd.AddCommand(MavenBuildCommand())
rootCmd.AddCommand(MavenExecuteStaticCodeChecksCommand())
rootCmd.AddCommand(NexusUploadCommand())
rootCmd.AddCommand(NpmExecuteScriptsCommand())
rootCmd.AddCommand(GctsCreateRepositoryCommand())
rootCmd.AddCommand(MalwareExecuteScanCommand())

View File

@ -0,0 +1,30 @@
// +build integration
// can be execute with go test -tags=integration ./integration/...
package main
import (
"bytes"
"github.com/SAP/jenkins-library/pkg/command"
"github.com/stretchr/testify/assert"
"testing"
)
func TestNpmExecuteScripts(t *testing.T) {
cmd := command.Command{}
cmd.SetDir("testdata/TestNpmIntegration")
piperOptions := []string{
"npmExecuteScripts",
"--install",
"--runScripts=ci-build,ci-backend-unit-test",
}
var commandOutput bytes.Buffer
cmd.Stdout(&commandOutput)
cmd.Stderr(&commandOutput)
err := cmd.RunExecutable(getPiperExecutable(), piperOptions...)
assert.NoError(t, err, "Calling piper with arguments %v failed.", piperOptions)
assert.Contains(t, commandOutput.String(), "Discovered pre-configured npm registry https://example.com")
}

View File

@ -0,0 +1 @@
@sap:registry=https://example.com

View File

@ -0,0 +1,5 @@
{
"name": "a",
"version": "1.0.0",
"lockfileVersion": 1
}

View File

@ -0,0 +1,9 @@
{
"name": "a",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"ci-build": "echo build"
}
}

View File

@ -0,0 +1,44 @@
metadata:
name: npmExecuteScripts
aliases:
- name: executeNpm
description: Execute npm run scripts on all npm packages in a project
longDescription: |
Execute npm run scripts in all package json files, if they implement the scripts.
spec:
inputs:
params:
- name: install
type: bool
description: Run npm install or similar commands depending on the project structure.
scope:
- PARAMETERS
- STAGES
- STEPS
default: false
- name: runScripts
type: '[]string'
description: List of additional run scripts to execute from package.json.
scope:
- PARAMETERS
- STAGES
- STEPS
- name: defaultNpmRegistry
type: string
description: URL of the npm registry to use. Defaults to https://registry.npmjs.org/
scope:
- PARAMETERS
- STAGES
- STEPS
- name: sapNpmRegistry
type: string
description: The default npm registry URL to be used as the remote mirror for the SAP npm packages.
scope:
- PARAMETERS
- STAGES
- STEPS
default: https://npm.sap.com
containers:
- name: node
image: node:12-buster-slim
imagePullPolicy: Never

View File

@ -74,4 +74,11 @@ class DownloadCacheUtils {
script.writeFile file: globalSettingsFilePath, text: mavenSettings
return globalSettingsFilePath
}
static String getNpmRegistryUri(Script script) {
script.node('master') {
return "http://${script.env.DL_CACHE_HOSTNAME}:8081/repository/npm-proxy/"
}
return ""
}
}

View File

@ -132,6 +132,7 @@ public class CommonStepsTest extends BasePiperTest{
'xsDeploy', //implementing new golang pattern without fields
'cloudFoundryDeleteService', //implementing new golang pattern without fields
'cloudFoundryCreateServiceKey', //implementing new golang pattern without fields
'npmExecuteScripts', //implementing new golang pattern without fields
'mavenBuild', //implementing new golang pattern without fields
'mavenExecute', //implementing new golang pattern without fields
'mavenExecuteStaticCodeChecks', //implementing new golang pattern without fields

View File

@ -0,0 +1,23 @@
import com.sap.piper.DownloadCacheUtils
import groovy.transform.Field
import static com.sap.piper.Prerequisites.checkScript
import static groovy.json.JsonOutput.toJson
@Field String STEP_NAME = getClass().getName()
@Field String METADATA_FILE = 'metadata/npmExecuteScripts.yaml'
//Metadata maintained in file project://resources/metadata/npmExecuteScripts.yaml
void call(Map parameters = [:]) {
final script = checkScript(this, parameters) ?: this
// No credentials required/supported as of now
List credentials = []
parameters['dockerOptions'] = DownloadCacheUtils.getDockerOptions(script)
if (DownloadCacheUtils.isEnabled(script)) {
parameters['defaultNpmRegistry'] = DownloadCacheUtils.getNpmRegistryUri(script)
}
piperExecuteBin(parameters, STEP_NAME, METADATA_FILE, credentials)
}