sap-jenkins-library/pkg/gcs/gcsClient.go

package gcs

import (
	"context"
	"io"
	"os"
	"path/filepath"

	"cloud.google.com/go/storage"
	"github.com/SAP/jenkins-library/pkg/log"
	"github.com/pkg/errors"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"
)

// Client is an interface to mock gcsClient
type Client interface {
	UploadFile(bucketID string, sourcePath string, targetPath string) error
	DownloadFile(bucketID string, sourcePath string, targetPath string) error
	ListFiles(bucketID string) ([]string, error)
	Close() error
}

// gcsClient provides functions to interact with google cloud storage API
type gcsClient struct {
	context       context.Context
	envVars       []EnvVar
	client        storage.Client
	clientOptions []option.ClientOption
	openFile      func(name string) (io.ReadCloser, error)
	createFile    func(name string) (io.WriteCloser, error)
}

// EnvVar defines an  environment variable incl. information about a potential modification to the variable
type EnvVar struct {
	Name     string
	Value    string
	Modified bool
}

type gcsOption func(*gcsClient)

// WithEnvVars initializes env variables in gcsClient
func WithEnvVars(envVars []EnvVar) gcsOption {
	return func(g *gcsClient) {
		g.envVars = envVars
	}
}

// WithEnvVars initializes the openFile function in gcsClient
func WithOpenFileFunction(openFile func(name string) (io.ReadCloser, error)) gcsOption {
	return func(g *gcsClient) {
		g.openFile = openFile
	}
}

// WithEnvVars initializes the createFile function in gcsClient
func WithCreateFileFunction(createFile func(name string) (io.WriteCloser, error)) gcsOption {
	return func(g *gcsClient) {
		g.createFile = createFile
	}
}

// WithEnvVars initializes the Google Cloud Storage client options
func WithClientOptions(opts ...option.ClientOption) gcsOption {
	return func(g *gcsClient) {
		g.clientOptions = append(g.clientOptions, opts...)
	}
}

// Init intitializes the google cloud storage client
func NewClient(opts ...gcsOption) (*gcsClient, error) {
	var (
		defaultOpenFile   = openFileFromFS
		defaultCreateFile = createFileOnFS
	)

	ctx := context.Background()
	gcsClient := &gcsClient{
		context:    ctx,
		openFile:   defaultOpenFile,
		createFile: defaultCreateFile,
	}

	// options handling
	for _, opt := range opts {
		opt(gcsClient)
	}

	gcsClient.prepareEnv()
	client, err := storage.NewClient(ctx, gcsClient.clientOptions...)
	if err != nil {
		return nil, errors.Wrapf(err, "bucket connection failed: %v", err)
	}
	gcsClient.client = *client
	return gcsClient, nil
}

// UploadFile uploads a file into a google cloud storage bucket
func (g *gcsClient) UploadFile(bucketID string, sourcePath string, targetPath string) error {
	target := g.client.Bucket(bucketID).Object(targetPath).NewWriter(g.context)
	log.Entry().Debugf("uploading %v to %v", sourcePath, targetPath)
	sourceFile, err := g.openFile(sourcePath)
	if err != nil {
		return errors.Wrapf(err, "could not open source file: %v", err)
	}
	defer sourceFile.Close()

	if err := g.copy(sourceFile, target); err != nil {
		return errors.Wrapf(err, "upload failed: %v", err)
	}

	if err := target.Close(); err != nil {
		return errors.Wrapf(err, "closing bucket failed: %v", err)
	}
	return nil
}

// DownloadFile downloads a file from a google cloud storage bucket
func (g *gcsClient) DownloadFile(bucketID string, sourcePath string, targetPath string) error {
	log.Entry().Debugf("downloading %v to %v\n", sourcePath, targetPath)
	gcsReader, err := g.client.Bucket(bucketID).Object(sourcePath).NewReader(g.context)
	if err != nil {
		return errors.Wrapf(err, "could not open source file from a google cloud storage bucket: %v", err)
	}

	targetWriter, err := g.createFile(targetPath)
	if err != nil {
		return errors.Wrapf(err, "could not create target file: %v", err)
	}
	defer targetWriter.Close()

	if err := g.copy(gcsReader, targetWriter); err != nil {
		return errors.Wrapf(err, "download failed: %v", err)
	}
	if err := gcsReader.Close(); err != nil {
		return errors.Wrapf(err, "closing bucket failed: %v", err)
	}
	return nil
}

// ListFiles lists all files in certain GCS bucket
func (g *gcsClient) ListFiles(bucketID string) ([]string, error) {
	fileNames := []string{}
	it := g.client.Bucket(bucketID).Objects(g.context, nil)
	for {
		attrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return nil, err
		}
		fileNames = append(fileNames, attrs.Name)
	}
	return fileNames, nil
}

// Close closes the client and removes previously set environment variables
func (g *gcsClient) Close() error {
	if err := g.client.Close(); err != nil {
		return err
	}
	if err := g.cleanupEnv(); err != nil {
		return err
	}
	return nil
}

func (g *gcsClient) copy(source io.Reader, target io.Writer) error {
	if _, err := io.Copy(target, source); err != nil {
		return err
	}
	return nil
}

// prepareEnv sets required environment variables in case they are not set yet
func (g *gcsClient) prepareEnv() {
	for key, env := range g.envVars {
		g.envVars[key].Modified = setenvIfEmpty(env.Name, env.Value)
	}
}

// cleanupEnv removes environment variables set by prepareEnv
func (g *gcsClient) cleanupEnv() error {
	for _, env := range g.envVars {
		if err := removeEnvIfPreviouslySet(env.Name, env.Modified); err != nil {
			return err
		}
	}
	return nil
}

func setenvIfEmpty(env, val string) bool {
	if len(os.Getenv(env)) == 0 {
		os.Setenv(env, val)
		return true
	}
	return false
}

func removeEnvIfPreviouslySet(env string, previouslySet bool) error {
	if previouslySet {
		if err := os.Setenv(env, ""); err != nil {
			return err
		}
	}
	return nil
}

func openFileFromFS(name string) (io.ReadCloser, error) {
	return os.Open(name)
}

func createFileOnFS(name string) (io.WriteCloser, error) {
	if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil {
		return nil, err
	}
	return os.Create(name)
}