From 3c19c364bde3120b8da88a2e36b9ccdc3641eb4e Mon Sep 17 00:00:00 2001 From: Akshay Pratinav Date: Tue, 12 Mar 2019 21:24:47 -0700 Subject: [PATCH 1/8] add pagination support for /user/teams --- providers/github.go | 72 +++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/providers/github.go b/providers/github.go index d39ee2b6..fc3740c4 100644 --- a/providers/github.go +++ b/providers/github.go @@ -137,36 +137,56 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { } `json:"organization"` } - params := url.Values{ - "limit": {"200"}, + type teamsPage []struct { + Name string `json:"name"` + Slug string `json:"slug"` + Org struct { + Login string `json:"login"` + } `json:"organization"` } - endpoint := &url.URL{ - Scheme: p.ValidateURL.Scheme, - Host: p.ValidateURL.Host, - Path: path.Join(p.ValidateURL.Path, "/user/teams"), - RawQuery: params.Encode(), - } - req, _ := http.NewRequest("GET", endpoint.String(), nil) - req.Header.Set("Accept", "application/vnd.github.v3+json") - req.Header.Set("Authorization", fmt.Sprintf("token %s", accessToken)) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return false, err - } + pn := 1 + for { + params := url.Values{ + "limit": {"200"}, + "page": {strconv.Itoa(pn)}, + } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return false, err - } - if resp.StatusCode != 200 { - return false, fmt.Errorf( - "got %d from %q %s", resp.StatusCode, endpoint.String(), body) - } + endpoint := &url.URL{ + Scheme: p.ValidateURL.Scheme, + Host: p.ValidateURL.Host, + Path: path.Join(p.ValidateURL.Path, "/user/teams"), + RawQuery: params.Encode(), + } - if err := json.Unmarshal(body, &teams); err != nil { - return false, fmt.Errorf("%s unmarshaling %s", err, body) + req, _ := http.NewRequest("GET", endpoint.String(), nil) + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("Authorization", fmt.Sprintf("token %s", accessToken)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, err + } + + body, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return false, err + } + if resp.StatusCode != 200 { + return false, fmt.Errorf( + "got %d from %q %s", resp.StatusCode, endpoint.String(), body) + } + + var tp teamsPage + if err := json.Unmarshal(body, &tp); err != nil { + return false, fmt.Errorf("%s unmarshaling %s", err, body) + } + if len(tp) == 0 { + break + } + + teams = append(teams, tp...) + pn++ } var hasOrg bool From e73f6501f0171e63f804c45469e58ac219a83b03 Mon Sep 17 00:00:00 2001 From: Akshay Pratinav Date: Thu, 14 Mar 2019 20:04:45 -0700 Subject: [PATCH 2/8] limit => per_page --- providers/github.go | 8 ++++---- providers/github_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/providers/github.go b/providers/github.go index fc3740c4..eecf5dfc 100644 --- a/providers/github.go +++ b/providers/github.go @@ -73,8 +73,8 @@ func (p *GitHubProvider) hasOrg(accessToken string) (bool, error) { pn := 1 for { params := url.Values{ - "limit": {"200"}, - "page": {strconv.Itoa(pn)}, + "per_page": {"200"}, + "page": {strconv.Itoa(pn)}, } endpoint := &url.URL{ @@ -148,8 +148,8 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { pn := 1 for { params := url.Values{ - "limit": {"200"}, - "page": {strconv.Itoa(pn)}, + "per_page": {"200"}, + "page": {strconv.Itoa(pn)}, } endpoint := &url.URL{ diff --git a/providers/github_test.go b/providers/github_test.go index c96877cb..1a66a59c 100644 --- a/providers/github_test.go +++ b/providers/github_test.go @@ -31,7 +31,7 @@ func testGitHubBackend(payload []string) *httptest.Server { pathToQueryMap := map[string][]string{ "/user": {""}, "/user/emails": {""}, - "/user/orgs": {"limit=200&page=1", "limit=200&page=2", "limit=200&page=3"}, + "/user/orgs": {"page=1&per_page=200", "page=2&per_page=200", "page=3&per_page=200"}, } return httptest.NewServer(http.HandlerFunc( From 6d15fe004eabcad382737976d3b686db7132becf Mon Sep 17 00:00:00 2001 From: Akshay Pratinav Date: Fri, 15 Mar 2019 08:00:20 -0700 Subject: [PATCH 3/8] change per_page value from 200 to 100 --- providers/github.go | 4 ++-- providers/github_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/providers/github.go b/providers/github.go index eecf5dfc..7bbc8b95 100644 --- a/providers/github.go +++ b/providers/github.go @@ -73,7 +73,7 @@ func (p *GitHubProvider) hasOrg(accessToken string) (bool, error) { pn := 1 for { params := url.Values{ - "per_page": {"200"}, + "per_page": {"100"}, "page": {strconv.Itoa(pn)}, } @@ -148,7 +148,7 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { pn := 1 for { params := url.Values{ - "per_page": {"200"}, + "per_page": {"100"}, "page": {strconv.Itoa(pn)}, } diff --git a/providers/github_test.go b/providers/github_test.go index 1a66a59c..3d63551e 100644 --- a/providers/github_test.go +++ b/providers/github_test.go @@ -31,7 +31,7 @@ func testGitHubBackend(payload []string) *httptest.Server { pathToQueryMap := map[string][]string{ "/user": {""}, "/user/emails": {""}, - "/user/orgs": {"page=1&per_page=200", "page=2&per_page=200", "page=3&per_page=200"}, + "/user/orgs": {"page=1&per_page=100", "page=2&per_page=100", "page=3&per_page=100"}, } return httptest.NewServer(http.HandlerFunc( From 31d7b61cc4816a20e372064d95df44cee0a38193 Mon Sep 17 00:00:00 2001 From: toshi-miura Date: Mon, 7 Oct 2019 22:38:57 +0900 Subject: [PATCH 4/8] Added handling of link header in githubAPI paging process ====================================================== changelog note [#274](https://github.com/pusher/oauth2_proxy/pull/274) Add github api pagination support (@toshi-miura ,@apratina) ====================================================== I didn't edit CHANGELOG.md. Since # 102 was taken over and the change difference of CHANGELOG.md was large --- providers/github.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/providers/github.go b/providers/github.go index 3dd677e9..dde166e9 100644 --- a/providers/github.go +++ b/providers/github.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "path" + "regexp" "strconv" "strings" @@ -148,6 +149,7 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { } pn := 1 + last := 0 for { params := url.Values{ "per_page": {"100"}, @@ -170,6 +172,30 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { } body, err := ioutil.ReadAll(resp.Body) + + if last == 0 { + // link header may not be obtained + // When paging is not required and all data can be retrieved with a single call + + // Conditions for obtaining the link header. + // 1. When paging is required (Example: When the data size is 100 and the page size is 99 or less) + // 2. When it exceeds the paging frame (Example: When there is only 10 records but the second page is called with a page size of 100) + + // link header at not last page + // ; rel="prev", ; rel="last", ; rel="first" + // link header at last page (doesn't exist last info) + // ; rel="prev", ; rel="first" + + link := resp.Header.Get("Link") + rep1 := regexp.MustCompile(`(?s).*\; rel="last".*`) + i, converr := strconv.Atoi(rep1.ReplaceAllString(link, "$1")) + + // If the last page cannot be taken from the link in the http header, the last variable remains zero + if converr == nil { + last = i + } + } + resp.Body.Close() if err != nil { return false, err @@ -188,6 +214,14 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { } teams = append(teams, tp...) + + if pn == last { + break + } + if last == 0 { + break + } + pn++ } From e71797b40911b87c308214219201343fdb3c2c61 Mon Sep 17 00:00:00 2001 From: toshi-miura Date: Fri, 11 Oct 2019 16:02:04 +0900 Subject: [PATCH 5/8] =?UTF-8?q?ReadAll()=20&=20Close()=20=E3=80=80close=20?= =?UTF-8?q?together.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- providers/github.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/providers/github.go b/providers/github.go index dde166e9..f9821cb2 100644 --- a/providers/github.go +++ b/providers/github.go @@ -171,8 +171,6 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { return false, err } - body, err := ioutil.ReadAll(resp.Body) - if last == 0 { // link header may not be obtained // When paging is not required and all data can be retrieved with a single call @@ -196,10 +194,13 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) { } } - resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) if err != nil { + resp.Body.Close() return false, err } + resp.Body.Close() + if resp.StatusCode != 200 { return false, fmt.Errorf( "got %d from %q %s", resp.StatusCode, endpoint.String(), body) From 0d256a329fb712da126319a250f236a45081147c Mon Sep 17 00:00:00 2001 From: toshi-miura Date: Fri, 11 Oct 2019 19:49:08 +0900 Subject: [PATCH 6/8] add change log. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44cd0cf6..ec03406e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Changes since v4.0.0 +- [#274](https://github.com/pusher/oauth2_proxy/pull/274) Supports many github teams with api pagination support (@toshi-miura ,@apratina) + # v4.0.0 ## Release Highlights From 610ee6d0ec5c03adef744a6c8a5a723842ca3893 Mon Sep 17 00:00:00 2001 From: toshi-miura Date: Sat, 12 Oct 2019 02:30:58 +0900 Subject: [PATCH 7/8] Fix typo. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc24e136..10aba308 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - [#227](https://github.com/pusher/oauth2_proxy/pull/227) Add Keycloak provider (@Ofinka) - [#273](https://github.com/pusher/oauth2_proxy/pull/273) Support Go 1.13 (@dio) - [#275](https://github.com/pusher/oauth2_proxy/pull/275) docker: build from debian buster (@syscll) -- [#274](https://github.com/pusher/oauth2_proxy/pull/274) Supports many github teams with api pagination support (@toshi-miura ,@apratina) +- [#274](https://github.com/pusher/oauth2_proxy/pull/274) Supports many github teams with api pagination support (@toshi-miura,@apratina) # v4.0.0 From e34f18ef2cf864c952e35977d9ee7e8d4ff19f8e Mon Sep 17 00:00:00 2001 From: toshi-miura Date: Mon, 21 Oct 2019 18:14:01 +0900 Subject: [PATCH 8/8] Update CHANGELOG.md Co-Authored-By: Joel Speed --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10aba308..201fb912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - [#227](https://github.com/pusher/oauth2_proxy/pull/227) Add Keycloak provider (@Ofinka) - [#273](https://github.com/pusher/oauth2_proxy/pull/273) Support Go 1.13 (@dio) - [#275](https://github.com/pusher/oauth2_proxy/pull/275) docker: build from debian buster (@syscll) -- [#274](https://github.com/pusher/oauth2_proxy/pull/274) Supports many github teams with api pagination support (@toshi-miura,@apratina) +- [#274](https://github.com/pusher/oauth2_proxy/pull/274) Supports many github teams with api pagination support (@toshi-miura, @apratina) # v4.0.0