Merge pull request #274 from toshi-miura/ap-gh-pagination-with-lastpage

Added pagination support for /user/teams github api with link header .

CHANGELOG.md

@@ -9,10 +9,12 @@
 - [#258](https://github.com/pusher/oauth2_proxy/pull/258) Add IDToken for Azure provider
   - This PR adds the IDToken into the session for the Azure provider allowing requests to a backend to be identified as a specific user. As a consequence, if you are using a cookie to store the session the cookie will now exceed the 4kb size limit and be split into multiple cookies. This can cause problems when using nginx as a proxy, resulting in no cookie being passed at all. Either increase the proxy_buffer_size in nginx or implement the redis session storage (see https://pusher.github.io/oauth2_proxy/configuration#redis-storage)
 - [#286](https://github.com/pusher/oauth2_proxy/pull/286) Requests.go updated with useful error messages (@biotom)
+- [#274](https://github.com/pusher/oauth2_proxy/pull/274) Supports many github teams with api pagination support (@toshi-miura, @apratina)
 - [#302](https://github.com/pusher/oauth2_proxy/pull/302) Rewrite dist script (@syscll)
 - [#304](https://github.com/pusher/oauth2_proxy/pull/304) Add new Logo! :tada: (@JoelSpeed)
 - [#300](https://github.com/pusher/oauth2_proxy/pull/300) Added userinfo endpoint (@kbabuadze)
 - [#309](https://github.com/pusher/oauth2_proxy/pull/309) Added support for custom CA when connecting to Redis cache
+
 # v4.0.0
 
 - [#248](https://github.com/pusher/oauth2_proxy/pull/248) Fix issue with X-Auth-Request-Redirect header being ignored

providers/github.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -75,7 +76,7 @@ func (p *GitHubProvider) hasOrg(accessToken string) (bool, error) {
 	pn := 1
 	for {
 		params := url.Values{
-			"limit": {"200"},
+			"per_page": {"100"},
 			"page":     {strconv.Itoa(pn)},
 		}
 
@@ -139,8 +140,20 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) {
 		} `json:"organization"`
 	}
 
+	type teamsPage []struct {
+		Name string `json:"name"`
+		Slug string `json:"slug"`
+		Org  struct {
+			Login string `json:"login"`
+		} `json:"organization"`
+	}
+
+	pn := 1
+	last := 0
+	for {
 		params := url.Values{
-		"limit": {"200"},
+			"per_page": {"100"},
+			"page":     {strconv.Itoa(pn)},
 		}
 
 		endpoint := &url.URL{
@@ -149,6 +162,7 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) {
 			Path:     path.Join(p.ValidateURL.Path, "/user/teams"),
 			RawQuery: params.Encode(),
 		}
+
 		req, _ := http.NewRequest("GET", endpoint.String(), nil)
 		req.Header.Set("Accept", "application/vnd.github.v3+json")
 		req.Header.Set("Authorization", fmt.Sprintf("token %s", accessToken))
@@ -157,19 +171,60 @@ func (p *GitHubProvider) hasOrgAndTeam(accessToken string) (bool, error) {
 			return false, err
 		}
 
+		if last == 0 {
+			// link header may not be obtained
+			// When paging is not required and all data can be retrieved with a single call
+
+			// Conditions for obtaining the link header.
+			// 1. When paging is required (Example: When the data size is 100 and the page size is 99 or less)
+			// 2. When it exceeds the paging frame (Example: When there is only 10 records but the second page is called with a page size of 100)
+
+			// link header at not last page
+			// <https://api.github.com/user/teams?page=1&per_page=100>; rel="prev", <https://api.github.com/user/teams?page=1&per_page=100>; rel="last", <https://api.github.com/user/teams?page=1&per_page=100>; rel="first"
+			// link header at last page (doesn't exist last info)
+			// <https://api.github.com/user/teams?page=3&per_page=10>; rel="prev", <https://api.github.com/user/teams?page=1&per_page=10>; rel="first"
+
+			link := resp.Header.Get("Link")
+			rep1 := regexp.MustCompile(`(?s).*\<https://api.github.com/user/teams\?page=(.)&per_page=[0-9]+\>; rel="last".*`)
+			i, converr := strconv.Atoi(rep1.ReplaceAllString(link, "$1"))
+
+			// If the last page cannot be taken from the link in the http header, the last variable remains zero
+			if converr == nil {
+				last = i
+			}
+		}
+
 		body, err := ioutil.ReadAll(resp.Body)
-	resp.Body.Close()
 		if err != nil {
+			resp.Body.Close()
 			return false, err
 		}
+		resp.Body.Close()
+
 		if resp.StatusCode != 200 {
 			return false, fmt.Errorf(
 				"got %d from %q %s", resp.StatusCode, endpoint.String(), body)
 		}
 
-	if err := json.Unmarshal(body, &teams); err != nil {
+		var tp teamsPage
+		if err := json.Unmarshal(body, &tp); err != nil {
 			return false, fmt.Errorf("%s unmarshaling %s", err, body)
 		}
+		if len(tp) == 0 {
+			break
+		}
+
+		teams = append(teams, tp...)
+
+		if pn == last {
+			break
+		}
+		if last == 0 {
+			break
+		}
+
+		pn++
+	}
 
 	var hasOrg bool
 	presentOrgs := make(map[string]bool)

providers/github_test.go

@@ -32,7 +32,7 @@ func testGitHubBackend(payload []string) *httptest.Server {
 	pathToQueryMap := map[string][]string{
 		"/user":        {""},
 		"/user/emails": {""},
-		"/user/orgs":   {"limit=200&page=1", "limit=200&page=2", "limit=200&page=3"},
+		"/user/orgs":   {"page=1&per_page=100", "page=2&per_page=100", "page=3&per_page=100"},
 	}
 
 	return httptest.NewServer(http.HandlerFunc(