1
0
mirror of https://github.com/SAP/jenkins-library.git synced 2024-12-14 11:03:09 +02:00
sap-jenkins-library/pkg/http/http.go

457 lines
15 KiB
Go
Raw Normal View History

package http
import (
2020-01-14 11:29:50 +02:00
"bytes"
"context"
"crypto/tls"
"encoding/json"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
2020-01-14 11:29:50 +02:00
"mime/multipart"
"net"
"net/http"
2020-01-14 11:29:50 +02:00
"os"
"strings"
"time"
"github.com/SAP/jenkins-library/pkg/log"
"github.com/hashicorp/go-retryablehttp"
"github.com/motemen/go-nuts/roundtime"
"github.com/pkg/errors"
2020-01-14 11:29:50 +02:00
"github.com/sirupsen/logrus"
)
// Client defines an http client object
type Client struct {
maxRequestDuration time.Duration
maxRetries int
transportTimeout time.Duration
transportSkipVerification bool
username string
password string
token string
logger *logrus.Entry
cookieJar http.CookieJar
doLogRequestBodyOnDebug bool
doLogResponseBodyOnDebug bool
useDefaultTransport bool
}
// ClientOptions defines the options to be set on the client
type ClientOptions struct {
// MaxRequestDuration has a default value of "0", meaning "no maximum
// request duration". If it is greater than 0, an overall, hard timeout
// for the request will be enforced. This should only be used if the
// length of the request bodies is known.
MaxRequestDuration time.Duration
MaxRetries int
// TransportTimeout defaults to 3 minutes, if not specified. It is
// used for the transport layer and duration of handshakes and such.
TransportTimeout time.Duration
TransportSkipVerification bool
Username string
Password string
Token string
Logger *logrus.Entry
CookieJar http.CookieJar
DoLogRequestBodyOnDebug bool
DoLogResponseBodyOnDebug bool
UseDefaultTransport bool
}
// TransportWrapper is a wrapper for central logging capabilities
type TransportWrapper struct {
Transport http.RoundTripper
doLogRequestBodyOnDebug bool
doLogResponseBodyOnDebug bool
}
// UploadRequestData encapsulates the parameters for calling uploader.Upload()
type UploadRequestData struct {
// Method is the HTTP method used for the request. Must be one of http.MethodPost or http.MethodPut.
Method string
// URL for the request
URL string
// File path to be stored in the created form field.
File string
// Form field name under which the file name will be stored.
FileFieldName string
// Additional form fields which will be added to the request if not nil.
FormFields map[string]string
// Reader from which the file contents will be read.
FileContent io.Reader
Header http.Header
Cookies []*http.Cookie
}
2020-01-14 11:29:50 +02:00
// Sender provides an interface to the piper http client for uid/pwd and token authenticated requests
type Sender interface {
SendRequest(method, url string, body io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error)
SetOptions(options ClientOptions)
}
2020-01-14 11:29:50 +02:00
// Uploader provides an interface to the piper http client for uid/pwd and token authenticated requests with upload capabilities
type Uploader interface {
Sender
2020-01-22 15:22:04 +02:00
UploadRequest(method, url, file, fieldName string, header http.Header, cookies []*http.Cookie) (*http.Response, error)
2020-01-14 11:29:50 +02:00
UploadFile(url, file, fieldName string, header http.Header, cookies []*http.Cookie) (*http.Response, error)
Upload(data UploadRequestData) (*http.Response, error)
2020-01-14 11:29:50 +02:00
}
// UploadFile uploads a file's content as multipart-form POST request to the specified URL
func (c *Client) UploadFile(url, file, fileFieldName string, header http.Header, cookies []*http.Cookie) (*http.Response, error) {
return c.UploadRequest(http.MethodPost, url, file, fileFieldName, header, cookies)
2020-01-22 15:22:04 +02:00
}
// UploadRequest uploads a file's content as multipart-form with given http method request to the specified URL
func (c *Client) UploadRequest(method, url, file, fileFieldName string, header http.Header, cookies []*http.Cookie) (*http.Response, error) {
fileHandle, err := os.Open(file)
if err != nil {
return &http.Response{}, errors.Wrapf(err, "unable to locate file %v", file)
}
defer fileHandle.Close()
return c.Upload(UploadRequestData{
Method: method,
URL: url,
File: file,
FileFieldName: fileFieldName,
FileContent: fileHandle,
Header: header,
Cookies: cookies,
})
}
// Upload uploads a file's content as multipart-form with given http method request to the specified URL
func (c *Client) Upload(data UploadRequestData) (*http.Response, error) {
if data.Method != http.MethodPost && data.Method != http.MethodPut {
return nil, errors.New(fmt.Sprintf("Http method %v is not allowed. Possible values are %v or %v", data.Method, http.MethodPost, http.MethodPut))
}
2020-01-14 11:29:50 +02:00
bodyBuffer := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuffer)
if data.FormFields != nil {
for fieldName, fieldValue := range data.FormFields {
err := bodyWriter.WriteField(fieldName, fieldValue)
if err != nil {
return &http.Response{}, errors.Wrapf(err, "error writing form field %v with value %v", fieldName, fieldValue)
}
}
2020-01-14 11:29:50 +02:00
}
fileWriter, err := bodyWriter.CreateFormFile(data.FileFieldName, data.File)
2020-01-14 11:29:50 +02:00
if err != nil {
return &http.Response{}, errors.Wrapf(err, "error creating form file %v for field %v", data.File, data.FileFieldName)
2020-01-14 11:29:50 +02:00
}
_, err = io.Copy(fileWriter, data.FileContent)
2020-01-14 11:29:50 +02:00
if err != nil {
return &http.Response{}, errors.Wrapf(err, "unable to copy file content of %v into request body", data.File)
2020-01-14 11:29:50 +02:00
}
err = bodyWriter.Close()
request, err := c.createRequest(data.Method, data.URL, bodyBuffer, &data.Header, data.Cookies)
2020-01-14 11:29:50 +02:00
if err != nil {
c.logger.Debugf("New %v request to %v", data.Method, data.URL)
return &http.Response{}, errors.Wrapf(err, "error creating %v request to %v", data.Method, data.URL)
2020-01-14 11:29:50 +02:00
}
startBoundary := strings.Index(bodyWriter.FormDataContentType(), "=") + 1
boundary := bodyWriter.FormDataContentType()[startBoundary:]
request.Header.Add("Content-Type", "multipart/form-data; boundary=\""+boundary+"\"")
request.Header.Add("Connection", "Keep-Alive")
return c.Send(request)
2020-01-14 11:29:50 +02:00
}
// SendRequest sends an http request with a defined method
//
// On error, any Response can be ignored and the Response.Body
// does not need to be closed.
func (c *Client) SendRequest(method, url string, body io.Reader, header http.Header, cookies []*http.Cookie) (*http.Response, error) {
2020-01-14 11:29:50 +02:00
request, err := c.createRequest(method, url, body, &header, cookies)
if err != nil {
return &http.Response{}, errors.Wrapf(err, "error creating %v request to %v", method, url)
}
return c.Send(request)
}
// Send sends an http request
func (c *Client) Send(request *http.Request) (*http.Response, error) {
httpClient := c.initialize()
2020-01-14 11:29:50 +02:00
response, err := httpClient.Do(request)
if err != nil {
return response, errors.Wrapf(err, "HTTP %v request to %v failed", request.Method, request.URL)
2020-01-14 11:29:50 +02:00
}
return c.handleResponse(response, request.URL.String())
2020-01-14 11:29:50 +02:00
}
// SetOptions sets options used for the http client
func (c *Client) SetOptions(options ClientOptions) {
c.doLogRequestBodyOnDebug = options.DoLogRequestBodyOnDebug
c.doLogResponseBodyOnDebug = options.DoLogResponseBodyOnDebug
c.useDefaultTransport = options.UseDefaultTransport
c.transportTimeout = options.TransportTimeout
c.transportSkipVerification = options.TransportSkipVerification
c.maxRequestDuration = options.MaxRequestDuration
2020-01-14 11:29:50 +02:00
c.username = options.Username
c.password = options.Password
c.token = options.Token
c.maxRetries = options.MaxRetries
if options.Logger != nil {
c.logger = options.Logger
} else {
c.logger = log.Entry().WithField("package", "SAP/jenkins-library/pkg/http")
}
c.cookieJar = options.CookieJar
2020-01-14 11:29:50 +02:00
}
func (c *Client) initialize() *http.Client {
c.applyDefaults()
2020-01-29 14:17:54 +02:00
c.logger = log.Entry().WithField("package", "SAP/jenkins-library/pkg/http")
var transport = &TransportWrapper{
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: c.transportTimeout,
}).DialContext,
ResponseHeaderTimeout: c.transportTimeout,
ExpectContinueTimeout: c.transportTimeout,
TLSHandshakeTimeout: c.transportTimeout,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: c.transportSkipVerification,
},
},
doLogRequestBodyOnDebug: c.doLogRequestBodyOnDebug,
doLogResponseBodyOnDebug: c.doLogResponseBodyOnDebug,
}
var httpClient *http.Client
if c.maxRetries > 0 {
retryClient := retryablehttp.NewClient()
retryClient.HTTPClient.Timeout = c.maxRequestDuration
retryClient.HTTPClient.Jar = c.cookieJar
retryClient.RetryMax = c.maxRetries
if !c.useDefaultTransport {
retryClient.HTTPClient.Transport = transport
}
retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
if err != nil && strings.Contains(err.Error(), "timeout") {
// Assuming timeouts could be retried
return true, nil
}
return retryablehttp.DefaultRetryPolicy(ctx, resp, err)
}
httpClient = retryClient.StandardClient()
} else {
httpClient = &http.Client{}
httpClient.Timeout = c.maxRequestDuration
httpClient.Jar = c.cookieJar
if !c.useDefaultTransport {
httpClient.Transport = transport
}
}
if c.transportSkipVerification {
c.logger.Debugf("TLS verification disabled")
}
c.logger.Debugf("Transport timeout: %v, max request duration: %v", c.transportTimeout, c.maxRequestDuration)
2020-01-14 11:29:50 +02:00
return httpClient
}
type contextKey struct {
name string
}
var contextKeyRequestStart = &contextKey{"RequestStart"}
// RoundTrip is the core part of this module and implements http.RoundTripper.
// Executes HTTP request with request/response logging.
func (t *TransportWrapper) RoundTrip(req *http.Request) (*http.Response, error) {
ctx := context.WithValue(req.Context(), contextKeyRequestStart, time.Now())
req = req.WithContext(ctx)
t.logRequest(req)
resp, err := t.Transport.RoundTrip(req)
t.logResponse(resp)
return resp, err
}
func (t *TransportWrapper) logRequest(req *http.Request) {
log.Entry().Debug("--------------------------------")
log.Entry().Debugf("--> %v request to %v", req.Method, req.URL)
log.Entry().Debugf("headers: %v", transformHeaders(req.Header))
log.Entry().Debugf("cookies: %v", transformCookies(req.Cookies()))
if t.doLogRequestBodyOnDebug {
log.Entry().Debugf("body: %v", transformBody(req.Body))
}
log.Entry().Debug("--------------------------------")
}
func (t *TransportWrapper) logResponse(resp *http.Response) {
if resp != nil {
ctx := resp.Request.Context()
if start, ok := ctx.Value(contextKeyRequestStart).(time.Time); ok {
log.Entry().Debugf("<-- response %v %v (%v)", resp.StatusCode, resp.Request.URL, roundtime.Duration(time.Now().Sub(start), 2))
} else {
log.Entry().Debugf("<-- response %v %v", resp.StatusCode, resp.Request.URL)
}
if t.doLogResponseBodyOnDebug {
log.Entry().Debugf("body: %v", transformBody(resp.Body))
}
} else {
log.Entry().Debug("response <nil>")
}
log.Entry().Debug("--------------------------------")
}
func transformHeaders(header http.Header) http.Header {
var h http.Header = map[string][]string{}
for name, value := range header {
if name == "Authorization" {
for _, v := range value {
// The format of the Authorization header value is: <type> <cred>.
// We don't register the full string since only the part after
// the first token is the secret in the narrower sense (applies at
// least for basic auth)
log.RegisterSecret(strings.Join(strings.Split(v, " ")[1:], " "))
}
// Since
// 1.) The auth header type itself might serve as a vector for an
// intrusion
// 2.) We cannot make assumtions about the structure of the auth
// header value since that depends on the type, e.g. several tokens
// where only some of the tokens define the secret
// we hide the full auth header value anyway in order to be on the
// save side.
value = []string{"<set>"}
}
h[name] = value
}
return h
}
func transformCookies(cookies []*http.Cookie) string {
result := ""
for _, c := range cookies {
result = fmt.Sprintf("%v %v", result, c.String())
}
return result
}
func transformBody(body io.ReadCloser) string {
if body == nil {
return ""
}
buf := new(bytes.Buffer)
buf.ReadFrom(body)
return buf.String()
}
2020-01-14 11:29:50 +02:00
func (c *Client) createRequest(method, url string, body io.Reader, header *http.Header, cookies []*http.Cookie) (*http.Request, error) {
request, err := http.NewRequest(method, url, body)
if err != nil {
2020-01-14 11:29:50 +02:00
return &http.Request{}, err
}
if header != nil {
2020-01-14 11:29:50 +02:00
for name, headers := range *header {
for _, h := range headers {
request.Header.Add(name, h)
}
}
}
if cookies != nil {
for _, cookie := range cookies {
request.AddCookie(cookie)
}
}
if len(c.username) > 0 {
request.SetBasicAuth(c.username, c.password)
2020-01-14 11:29:50 +02:00
c.logger.Debug("Using Basic Authentication ****/****")
}
2020-01-14 11:29:50 +02:00
if len(c.token) > 0 {
request.Header.Add("Authorization", c.token)
}
2020-01-14 11:29:50 +02:00
return request, nil
}
func (c *Client) handleResponse(response *http.Response, url string) (*http.Response, error) {
// 2xx codes do not create an error
if response.StatusCode >= 200 && response.StatusCode < 300 {
return response, nil
}
switch response.StatusCode {
case http.StatusUnauthorized:
2020-01-14 11:29:50 +02:00
c.logger.WithField("HTTP Error", "401 (Unauthorized)").Error("Credentials invalid, please check your user credentials!")
case http.StatusForbidden:
2020-01-14 11:29:50 +02:00
c.logger.WithField("HTTP Error", "403 (Forbidden)").Error("Permission issue, please check your user permissions!")
case http.StatusNotFound:
c.logger.WithField("HTTP Error", "404 (Not Found)").Errorf("Requested resource ('%s') could not be found", url)
case http.StatusInternalServerError:
c.logger.WithField("HTTP Error", "500 (Internal Server Error)").Error("Unknown error occurred.")
}
return response, fmt.Errorf("Request to %v returned with response %v", response.Request.URL, response.Status)
}
func (c *Client) applyDefaults() {
if c.transportTimeout == 0 {
c.transportTimeout = 3 * time.Minute
}
if c.logger == nil {
c.logger = log.Entry().WithField("package", "SAP/jenkins-library/pkg/http")
}
}
// ParseHTTPResponseBodyXML parses a XML http response into a given interface
func ParseHTTPResponseBodyXML(resp *http.Response, response interface{}) error {
if resp == nil {
return errors.Errorf("cannot parse HTTP response with value <nil>")
}
bodyText, readErr := ioutil.ReadAll(resp.Body)
if readErr != nil {
return errors.Wrap(readErr, "HTTP response body could not be read")
}
marshalErr := xml.Unmarshal(bodyText, &response)
if marshalErr != nil {
return errors.Wrapf(marshalErr, "HTTP response body could not be parsed as XML: %v", string(bodyText))
}
return nil
}
// ParseHTTPResponseBodyJSON parses a JSON http response into a given interface
func ParseHTTPResponseBodyJSON(resp *http.Response, response interface{}) error {
if resp == nil {
return errors.Errorf("cannot parse HTTP response with value <nil>")
}
bodyText, readErr := ioutil.ReadAll(resp.Body)
if readErr != nil {
return errors.Wrapf(readErr, "HTTP response body could not be read")
}
marshalErr := json.Unmarshal(bodyText, &response)
if marshalErr != nil {
return errors.Wrapf(marshalErr, "HTTP response body could not be parsed as JSON: %v", string(bodyText))
}
return nil
}