diff --git a/cmd/artifactPrepareVersion.go b/cmd/artifactPrepareVersion.go index fcc2e38ad..f78b454da 100644 --- a/cmd/artifactPrepareVersion.go +++ b/cmd/artifactPrepareVersion.go @@ -497,10 +497,6 @@ func propagateVersion(config *artifactPrepareVersionOptions, utils artifactPrepa } for i, targetTool := range config.AdditionalTargetTools { - if targetTool == config.BuildTool { - // ignore configured build tool - continue - } var buildDescriptors []string if len(config.AdditionalTargetDescriptors) > 0 { diff --git a/cmd/npmExecuteScripts.go b/cmd/npmExecuteScripts.go index 05a7c8eea..170fd7c1b 100644 --- a/cmd/npmExecuteScripts.go +++ b/cmd/npmExecuteScripts.go @@ -79,14 +79,21 @@ func runNpmExecuteScripts(npmExecutor npm.Executor, config *npmExecuteScriptsOpt commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo if config.Publish { - packageJSONFiles, err := npmExecutor.FindPackageJSONFilesWithExcludes(config.BuildDescriptorExcludeList) - if err != nil { - return err - } + if len(config.BuildDescriptorList) > 0 { + err = npmExecutor.PublishAllPackages(config.BuildDescriptorList, config.RepositoryURL, config.RepositoryUsername, config.RepositoryPassword, config.PackBeforePublish) + if err != nil { + return err + } + } else { + packageJSONFiles, err := npmExecutor.FindPackageJSONFilesWithExcludes(config.BuildDescriptorExcludeList) + if err != nil { + return err + } - err = npmExecutor.PublishAllPackages(packageJSONFiles, config.RepositoryURL, config.RepositoryUsername, config.RepositoryPassword, config.PackBeforePublish) - if err != nil { - return err + err = npmExecutor.PublishAllPackages(packageJSONFiles, config.RepositoryURL, config.RepositoryUsername, config.RepositoryPassword, config.PackBeforePublish) + if err != nil { + return err + } } } diff --git a/cmd/npmExecuteScripts_generated.go b/cmd/npmExecuteScripts_generated.go index 3cc7b9589..007e5445f 100644 --- a/cmd/npmExecuteScripts_generated.go +++ b/cmd/npmExecuteScripts_generated.go @@ -238,7 +238,7 @@ func addNpmExecuteScriptsFlags(cmd *cobra.Command, stepConfig *npmExecuteScripts cmd.Flags().StringVar(&stepConfig.RepositoryPassword, "repositoryPassword", os.Getenv("PIPER_repositoryPassword"), "Password for the repository to which the project artifacts should be published.") cmd.Flags().StringVar(&stepConfig.RepositoryUsername, "repositoryUsername", os.Getenv("PIPER_repositoryUsername"), "Username for the repository to which the project artifacts should be published.") cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the npm build . This information is typically used for compliance related processes.") - cmd.Flags().BoolVar(&stepConfig.PackBeforePublish, "packBeforePublish", false, "used for executing npm pack first, followed by npm publish. This two step maybe required when you are building a scoped packages and have npm dependencies from the same scope") + cmd.Flags().BoolVar(&stepConfig.PackBeforePublish, "packBeforePublish", false, "used for executing npm pack first, followed by npm publish. This two step maybe required in two cases. case 1) When building multiple npm packages (multiple package.json) please keep this parameter true and also see `buildDescriptorList` or `buildDescriptorExcludeList` to choose which package(s) to publish. case 2)when you are building a single npm (single `package.json` in your repo) / multiple npm (multiple package.json) scoped package(s) and have npm dependencies from the same scope.") } diff --git a/pkg/mock/fileUtils.go b/pkg/mock/fileUtils.go index 414ec1edd..883ef46e7 100644 --- a/pkg/mock/fileUtils.go +++ b/pkg/mock/fileUtils.go @@ -671,3 +671,28 @@ func (f *FilesMock) Open(name string) (io.ReadWriteCloser, error) { func (f *FilesMock) Create(name string) (io.ReadWriteCloser, error) { return f.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666) } + +type FilesMockRelativeGlob struct { + *FilesMock +} + +// Glob of FilesMockRelativeGlob cuts current directory path part from files if pattern is relative +func (f *FilesMockRelativeGlob) Glob(pattern string) ([]string, error) { + var matches []string + if f.files == nil { + return matches, nil + } + for path := range f.files { + if !filepath.IsAbs(pattern) { + path = strings.TrimLeft(path, f.Separator+f.CurrentDir) + } + path = strings.TrimLeft(path, f.Separator) + matched, _ := doublestar.PathMatch(pattern, path) + if matched { + matches = append(matches, path) + } + } + // The order in f.files is not deterministic, this would result in flaky tests. + sort.Strings(matches) + return matches, nil +} diff --git a/pkg/npm/publish.go b/pkg/npm/publish.go index 76cd4a62b..b522f9d6c 100644 --- a/pkg/npm/publish.go +++ b/pkg/npm/publish.go @@ -55,6 +55,8 @@ func (exec *Execute) PublishAllPackages(packageJSONFiles []string, registry, use func (exec *Execute) publish(packageJSON, registry, username, password string, packBeforePublish bool) error { execRunner := exec.Utils.GetExecRunner() + oldWorkingDirectory, err := exec.Utils.Getwd() + scope, err := exec.readPackageScope(packageJSON) if err != nil { @@ -130,42 +132,42 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p } if packBeforePublish { - tmpDirectory, err := exec.Utils.TempDir(".", "temp-") - - if err != nil { - return errors.Wrap(err, "creating temp directory failed") + // change directory in package json file , since npm pack will run only for that packages + if err := exec.Utils.Chdir(filepath.Dir(packageJSON)); err != nil { + return fmt.Errorf("failed to change into directory for executing script: %w", err) } - defer exec.Utils.RemoveAll(tmpDirectory) + if err := execRunner.RunExecutable("npm", "pack"); err != nil { + return err + } - err = execRunner.RunExecutable("npm", "pack", "--pack-destination", tmpDirectory) + tarballs, err := exec.Utils.Glob(filepath.Join(".", "*.tgz")) if err != nil { return err } - _, err = exec.Utils.Copy(npmrc.filepath, filepath.Join(tmpDirectory, ".piperNpmrc")) - if err != nil { - return fmt.Errorf("error copying piperNpmrc file from %v to %v with error: %w", - npmrc.filepath, filepath.Join(tmpDirectory, ".piperNpmrc"), err) + // we do not maintain the tarball file name and hence expect only one tarball that comes + // from the npm pack command + if len(tarballs) < 1 { + return fmt.Errorf("no tarballs found") } - - tarballs, err := exec.Utils.Glob(filepath.Join(tmpDirectory, "*.tgz")) - - if err != nil { - return err - } - - if len(tarballs) != 1 { + if len(tarballs) > 1 { return fmt.Errorf("found more tarballs than expected: %v", tarballs) } tarballFilePath, err := exec.Utils.Abs(tarballs[0]) - if err != nil { return err } - projectNpmrc := filepath.Join(filepath.Dir(packageJSON), ".npmrc") + // if a user has a .npmrc file and if it has a scope (e.g @sap to download scoped dependencies) + // if the package to be published also has the same scope (@sap) then npm gets confused + // and tries to publish to the scope that comes from the npmrc file + // and is not the desired publish since we want to publish to the other registry (from .piperNpmrc) + // file and not to the one mentioned in the users npmrc file + // to solve this we rename the users npmrc file before publish, the original npmrc is already + // packaged in the tarball and hence renaming it before publish should not have an effect + projectNpmrc := filepath.Join(".", ".npmrc") projectNpmrcExists, _ := exec.Utils.FileExists(projectNpmrc) if projectNpmrcExists { @@ -176,7 +178,7 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p } } - err = execRunner.RunExecutable("npm", "publish", "--tarball", tarballFilePath, "--userconfig", filepath.Join(tmpDirectory, ".piperNpmrc"), "--registry", registry) + err = execRunner.RunExecutable("npm", "publish", "--tarball", tarballFilePath, "--userconfig", ".piperNpmrc", "--registry", registry) if err != nil { return errors.Wrap(err, "failed publishing artifact") } @@ -188,6 +190,10 @@ func (exec *Execute) publish(packageJSON, registry, username, password string, p log.Entry().Warnf("unable to rename the .npmrc file : %v", err) } } + + if err := exec.Utils.Chdir(oldWorkingDirectory); err != nil { + return fmt.Errorf("failed to change back into original directory: %w", err) + } } else { err := execRunner.RunExecutable("npm", "publish", "--userconfig", npmrc.filepath, "--registry", registry) if err != nil { diff --git a/pkg/npm/publish_test.go b/pkg/npm/publish_test.go index df92fa1e7..6ca7fdaaa 100644 --- a/pkg/npm/publish_test.go +++ b/pkg/npm/publish_test.go @@ -4,15 +4,31 @@ package npm import ( + "github.com/SAP/jenkins-library/pkg/mock" "io" "path/filepath" - "regexp" "testing" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/stretchr/testify/assert" ) +type npmMockUtilsBundleRelativeGlob struct { + *mock.FilesMockRelativeGlob + execRunner *mock.ExecMockRunner +} + +func (u *npmMockUtilsBundleRelativeGlob) GetExecRunner() ExecRunner { + return u.execRunner +} + +func newNpmMockUtilsBundleRelativeGlob() npmMockUtilsBundleRelativeGlob { + return npmMockUtilsBundleRelativeGlob{ + FilesMockRelativeGlob: &mock.FilesMockRelativeGlob{FilesMock: &mock.FilesMock{}}, + execRunner: &mock.ExecMockRunner{}, + } +} + func TestNpmPublish(t *testing.T) { type wants struct { publishConfigPath string @@ -102,9 +118,9 @@ func TestNpmPublish(t *testing.T) { packBeforePublish: true, wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nregistry=https://my.private.npm.registry/\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/package.tgz", }, }, { @@ -123,9 +139,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsThePassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "registry=https://my.private.npm.registry/\n//my.private.npm.registry/:_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/package.tgz", }, }, { @@ -145,9 +161,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsTheOtherPassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "//my.private.npm.registry/:_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nregistry=https://my.other.private.npm.registry/\n//my.other.private.npm.registry/:_auth=VGhpc0lzVGhlT3RoZXJVc2VyOkFuZEhlcmVJc1RoZU90aGVyUGFzc3dvcmQ=\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/package.tgz", }, }, // scoped project @@ -216,9 +232,9 @@ func TestNpmPublish(t *testing.T) { packBeforePublish: true, wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\n@piper:registry=https://my.private.npm.registry/\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/package.tgz", }, }, { @@ -237,9 +253,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsThePassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "registry=https://my.private.npm.registry/\n@piper:registry=https://my.private.npm.registry/\n//my.private.npm.registry/:_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/package.tgz", }, }, { @@ -259,9 +275,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsTheOtherPassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "//my.private.npm.registry/:_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nregistry=https://my.other.private.npm.registry/\n@piper:registry=https://my.other.private.npm.registry/\n//my.other.private.npm.registry/:_auth=VGhpc0lzVGhlT3RoZXJVc2VyOkFuZEhlcmVJc1RoZU90aGVyUGFzc3dvcmQ=\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/package.tgz", }, }, // project in a subfolder @@ -330,9 +346,9 @@ func TestNpmPublish(t *testing.T) { packBeforePublish: true, wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nregistry=https://my.private.npm.registry/\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/sub/package.tgz", }, }, { @@ -351,9 +367,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsThePassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "registry=https://my.private.npm.registry/\n//my.private.npm.registry/:_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/sub/package.tgz", }, }, { @@ -373,9 +389,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsTheOtherPassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "//my.private.npm.registry/:_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nregistry=https://my.other.private.npm.registry/\n//my.other.private.npm.registry/:_auth=VGhpc0lzVGhlT3RoZXJVc2VyOkFuZEhlcmVJc1RoZU90aGVyUGFzc3dvcmQ=\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/sub/package.tgz", }, }, // scoped project in a subfolder @@ -444,9 +460,9 @@ func TestNpmPublish(t *testing.T) { packBeforePublish: true, wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\n@piper:registry=https://my.private.npm.registry/\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/sub/package.tgz", }, }, { @@ -465,9 +481,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsThePassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "registry=https://my.private.npm.registry/\n@piper:registry=https://my.private.npm.registry/\n//my.private.npm.registry/:_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/sub/package.tgz", }, }, { @@ -487,9 +503,9 @@ func TestNpmPublish(t *testing.T) { registryPassword: "AndHereIsTheOtherPassword", wants: wants{ - publishConfigPath: `temp-(?:test|[0-9]+)/\.piperNpmrc`, + publishConfigPath: `\.piperNpmrc`, publishConfig: "_auth=VGhpc0lzVGhlVXNlcjpBbmRIZXJlSXNUaGVQYXNzd29yZA==\nregistry=https://my.other.private.npm.registry/\n@piper:registry=https://my.other.private.npm.registry/\n//my.other.private.npm.registry/:_auth=VGhpc0lzVGhlT3RoZXJVc2VyOkFuZEhlcmVJc1RoZU90aGVyUGFzc3dvcmQ=\nalways-auth=true\n", - tarballPath: "/temp-test/package.tgz", + tarballPath: "/sub/package.tgz", }, }, // TODO multiple projects @@ -497,17 +513,14 @@ func TestNpmPublish(t *testing.T) { for _, test := range tt { t.Run(test.name, func(t *testing.T) { - utils := newNpmMockUtilsBundle() - + utils := newNpmMockUtilsBundleRelativeGlob() for path, content := range test.files { utils.AddFile(path, []byte(content)) } - - options := ExecutorOptions{} + utils.Separator = string(filepath.Separator) exec := &Execute{ - Utils: &utils, - Options: options, + Utils: &utils, } propertiesLoadFile = utils.FileRead @@ -516,18 +529,8 @@ func TestNpmPublish(t *testing.T) { // This stub simulates the behavior of npm pack and puts a tgz into the requested utils.execRunner.Stub = func(call string, stdoutReturn map[string]string, shouldFailOnCommand map[string]error, stdout io.Writer) error { - r := regexp.MustCompile(`npm\s+pack\s+.*--pack-destination\s+(?P[^\s]+).*`) - - matches := r.FindStringSubmatch(call) - - if len(matches) == 0 { - return nil - } - - packDestination := matches[1] - - utils.AddFile(filepath.Join(packDestination, "package.tgz"), []byte("this is a tgz file")) - + //tgzTargetPath := filepath.Dir(test.packageDescriptors[0]) + utils.AddFile(filepath.Join(".", "package.tgz"), []byte("this is a tgz file")) return nil } @@ -543,16 +546,20 @@ func TestNpmPublish(t *testing.T) { if len(test.wants.tarballPath) > 0 && assert.Contains(t, publishCmd.Params, "--tarball") { tarballPath := publishCmd.Params[piperutils.FindString(publishCmd.Params, "--tarball")+1] - assert.Equal(t, test.wants.tarballPath, tarballPath) + assert.Equal(t, test.wants.tarballPath, filepath.ToSlash(tarballPath)) } if assert.Contains(t, publishCmd.Params, "--userconfig") { effectivePublishConfigPath := publishCmd.Params[piperutils.FindString(publishCmd.Params, "--userconfig")+1] - assert.Regexp(t, test.wants.publishConfigPath, effectivePublishConfigPath) + assert.Regexp(t, test.wants.publishConfigPath, filepath.ToSlash(effectivePublishConfigPath)) + + if test.packBeforePublish { + subPath := filepath.Dir(test.packageDescriptors[0]) + effectivePublishConfigPath = filepath.Join(subPath, effectivePublishConfigPath) + } effectiveConfig, err := utils.FileRead(effectivePublishConfigPath) - if assert.NoError(t, err) { assert.Equal(t, test.wants.publishConfig, string(effectiveConfig)) } diff --git a/resources/metadata/npmExecuteScripts.yaml b/resources/metadata/npmExecuteScripts.yaml index d76490da9..115cfb0ab 100644 --- a/resources/metadata/npmExecuteScripts.yaml +++ b/resources/metadata/npmExecuteScripts.yaml @@ -150,7 +150,7 @@ spec: - name: packBeforePublish type: bool default: false - description: used for executing npm pack first, followed by npm publish. This two step maybe required when you are building a scoped packages and have npm dependencies from the same scope + description: used for executing npm pack first, followed by npm publish. This two step maybe required in two cases. case 1) When building multiple npm packages (multiple package.json) please keep this parameter true and also see `buildDescriptorList` or `buildDescriptorExcludeList` to choose which package(s) to publish. case 2)when you are building a single npm (single `package.json` in your repo) / multiple npm (multiple package.json) scoped package(s) and have npm dependencies from the same scope. scope: - STEPS - STAGES