diff --git a/Makefile b/Makefile index 142530104..e73fee7b8 100644 --- a/Makefile +++ b/Makefile @@ -57,8 +57,8 @@ vendor: go mod tidy go mod vendor -format: - @gofmt -s -w ${GOFILES_NOVENDOR} +format: install-tools + @gofumpt -extra -w ${GOFILES_NOVENDOR} .PHONY: docs docs: @@ -140,6 +140,9 @@ install-tools: fi ; \ hash lint > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ go install github.com/rs/zerolog/cmd/lint@latest; \ + fi ; \ + hash gofumpt > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + go install mvdan.cc/gofumpt@latest; \ fi cross-compile-server: diff --git a/server/api/hook.go b/server/api/hook.go index 548dde88a..92f7e1c79 100644 --- a/server/api/hook.go +++ b/server/api/hook.go @@ -69,8 +69,9 @@ func BlockTilQueueHasRunningItem(c *gin.Context) { // PostHook start a pipeline triggered by a forges post webhook func PostHook(c *gin.Context) { _store := store.FromContext(c) + remote := server.Config.Services.Remote - tmpRepo, tmpBuild, err := server.Config.Services.Remote.Hook(c, c.Request) + tmpRepo, tmpBuild, err := remote.Hook(c, c.Request) if err != nil { msg := "failure to parse hook" log.Debug().Err(err).Msg(msg) @@ -100,7 +101,7 @@ func PostHook(c *gin.Context) { return } - repo, err := _store.GetRepoName(tmpRepo.Owner + "/" + tmpRepo.Name) + repo, err := _store.GetRepoNameFallback(tmpRepo.RemoteID, tmpRepo.FullName) if err != nil { msg := fmt.Sprintf("failure to get repo %s from store", tmpRepo.FullName) log.Error().Err(err).Msg(msg) @@ -114,6 +115,23 @@ func PostHook(c *gin.Context) { return } + oldFullName := repo.FullName + if oldFullName != tmpRepo.FullName { + // create a redirection + err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName}) + if err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) + return + } + } + + repo.Update(tmpRepo) + err = _store.UpdateRepo(repo) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + // get the token and verify the hook is authorized parsed, err := token.ParseRequest(c.Request, func(_ *token.Token) (string, error) { return repo.Hash, nil @@ -124,7 +142,18 @@ func PostHook(c *gin.Context) { c.String(http.StatusBadRequest, msg) return } - if parsed.Text != repo.FullName { + verifiedKey := parsed.Text == oldFullName + if !verifiedKey { + verifiedKey, err = _store.HasRedirectionForRepo(repo.ID, repo.FullName) + if err != nil { + msg := "failure to verify token from hook. Could not check for redirections of the repo" + log.Error().Err(err).Msg(msg) + c.String(http.StatusInternalServerError, msg) + return + } + } + + if !verifiedKey { msg := fmt.Sprintf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text) log.Debug().Msg(msg) c.String(http.StatusForbidden, msg) diff --git a/server/api/repo.go b/server/api/repo.go index 5b0f96b1c..10d09e539 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -86,17 +86,25 @@ func PostRepo(c *gin.Context) { sig, ) + from, err := remote.Repo(c, user, repo.RemoteID, repo.Owner, repo.Name) + if err == nil { + if repo.FullName != from.FullName { + // create a redirection + err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName}) + if err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) + return + } + } + repo.Update(from) + } + err = remote.Activate(c, user, repo, link) if err != nil { c.String(http.StatusInternalServerError, err.Error()) return } - from, err := remote.Repo(c, user, repo.Owner, repo.Name) - if err == nil { - repo.Update(from) - } - err = _store.UpdateRepo(repo) if err != nil { c.String(http.StatusInternalServerError, err.Error()) @@ -251,35 +259,36 @@ func RepairRepo(c *gin.Context) { sig, ) - if err := remote.Deactivate(c, user, repo, host); err != nil { - log.Trace().Err(err).Msgf("deactivate repo '%s' to repair failed", repo.FullName) - } - if err := remote.Activate(c, user, repo, link); err != nil { - c.String(500, err.Error()) - return - } - - from, err := remote.Repo(c, user, repo.Owner, repo.Name) + from, err := remote.Repo(c, user, repo.RemoteID, repo.Owner, repo.Name) if err != nil { log.Error().Err(err).Msgf("get repo '%s/%s' from remote", repo.Owner, repo.Name) c.AbortWithStatus(http.StatusInternalServerError) return } - repo.Name = from.Name - repo.Owner = from.Owner - repo.FullName = from.FullName - repo.Avatar = from.Avatar - repo.Link = from.Link - repo.Clone = from.Clone - repo.IsSCMPrivate = from.IsSCMPrivate - if repo.IsSCMPrivate != from.IsSCMPrivate { - repo.ResetVisibility() + + if repo.FullName != from.FullName { + // create a redirection + err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName}) + if err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) + return + } } + + repo.Update(from) if err := _store.UpdateRepo(repo); err != nil { _ = c.AbortWithError(http.StatusInternalServerError, err) return } + if err := remote.Deactivate(c, user, repo, host); err != nil { + log.Trace().Err(err).Msgf("deactivate repo '%s' to repair failed", repo.FullName) + } + if err := remote.Activate(c, user, repo, link); err != nil { + c.String(500, err.Error()) + return + } + c.Writer.WriteHeader(http.StatusOK) } @@ -302,7 +311,7 @@ func MoveRepo(c *gin.Context) { return } - from, err := remote.Repo(c, user, owner, name) + from, err := remote.Repo(c, user, "", owner, name) if err != nil { _ = c.AbortWithError(http.StatusInternalServerError, err) return @@ -312,17 +321,14 @@ func MoveRepo(c *gin.Context) { return } - repo.Name = from.Name - repo.Owner = from.Owner - repo.FullName = from.FullName - repo.Avatar = from.Avatar - repo.Link = from.Link - repo.Clone = from.Clone - repo.IsSCMPrivate = from.IsSCMPrivate - if repo.IsSCMPrivate != from.IsSCMPrivate { - repo.ResetVisibility() + err = _store.CreateRedirection(&model.Redirection{RepoID: repo.ID, FullName: repo.FullName}) + if err != nil { + _ = c.AbortWithError(http.StatusInternalServerError, err) + return } + repo.Update(from) + errStore := _store.UpdateRepo(repo) if errStore != nil { _ = c.AbortWithError(http.StatusInternalServerError, errStore) diff --git a/server/model/perm.go b/server/model/perm.go index 309848ad1..f139e76cf 100644 --- a/server/model/perm.go +++ b/server/model/perm.go @@ -25,15 +25,15 @@ type PermStore interface { // Perm defines a repository permission for an individual user. type Perm struct { - UserID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_user_id'"` - RepoID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_repo_id'"` - Repo string `json:"-" xorm:"-"` // TODO: better caching (use type *Repo) - Pull bool `json:"pull" xorm:"perm_pull"` - Push bool `json:"push" xorm:"perm_push"` - Admin bool `json:"admin" xorm:"perm_admin"` - Synced int64 `json:"synced" xorm:"perm_synced"` - Created int64 `json:"created" xorm:"created"` - Updated int64 `json:"updated" xorm:"updated"` + UserID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_user_id'"` + RepoID int64 `json:"-" xorm:"UNIQUE(s) INDEX NOT NULL 'perm_repo_id'"` + Repo *Repo `json:"-" xorm:"-"` + Pull bool `json:"pull" xorm:"perm_pull"` + Push bool `json:"push" xorm:"perm_push"` + Admin bool `json:"admin" xorm:"perm_admin"` + Synced int64 `json:"synced" xorm:"perm_synced"` + Created int64 `json:"created" xorm:"created"` + Updated int64 `json:"updated" xorm:"updated"` } // TableName return database table name for xorm diff --git a/server/model/redirection.go b/server/model/redirection.go new file mode 100644 index 000000000..5481d590b --- /dev/null +++ b/server/model/redirection.go @@ -0,0 +1,11 @@ +package model + +type Redirection struct { + ID int64 `xorm:"pk autoincr 'redirection_id'"` + RepoID int64 `xorm:"'repo_id'"` + FullName string `xorm:"UNIQUE INDEX 'repo_full_name'"` +} + +func (r Redirection) TableName() string { + return "redirections" +} diff --git a/server/model/repo.go b/server/model/repo.go index f72b98b1c..0c60b08a0 100644 --- a/server/model/repo.go +++ b/server/model/repo.go @@ -26,6 +26,7 @@ import ( type Repo struct { ID int64 `json:"id,omitempty" xorm:"pk autoincr 'repo_id'"` UserID int64 `json:"-" xorm:"repo_user_id"` + RemoteID RemoteID `json:"-" xorm:"'remote_id'"` Owner string `json:"owner" xorm:"UNIQUE(name) 'repo_owner'"` Name string `json:"name" xorm:"UNIQUE(name) 'repo_name'"` FullName string `json:"full_name" xorm:"UNIQUE 'repo_full_name'"` @@ -74,6 +75,10 @@ func ParseRepo(str string) (user, repo string, err error) { // Update updates the repository with values from the given Repo. func (r *Repo) Update(from *Repo) { + r.RemoteID = from.RemoteID + r.Owner = from.Owner + r.Name = from.Name + r.FullName = from.FullName r.Avatar = from.Avatar r.Link = from.Link r.SCMKind = from.SCMKind @@ -99,3 +104,9 @@ type RepoPatch struct { AllowPull *bool `json:"allow_pr,omitempty"` CancelPreviousPipelineEvents *[]WebhookEvent `json:"cancel_previous_pipeline_events"` } + +type RemoteID string + +func (r RemoteID) IsValid() bool { + return r != "" && r != "0" +} diff --git a/server/remote/bitbucket/bitbucket.go b/server/remote/bitbucket/bitbucket.go index e69cd5e2f..d6710f151 100644 --- a/server/remote/bitbucket/bitbucket.go +++ b/server/remote/bitbucket/bitbucket.go @@ -142,7 +142,10 @@ func (c *config) Teams(ctx context.Context, u *model.User) ([]*model.Team, error } // Repo returns the named Bitbucket repository. -func (c *config) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) { +func (c *config) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) { + if id.IsValid() { + name = string(id) + } repo, err := c.newClient(ctx, u).FindRepo(owner, name) if err != nil { return nil, err diff --git a/server/remote/bitbucket/bitbucket_test.go b/server/remote/bitbucket/bitbucket_test.go index 412b93e01..c3cb63182 100644 --- a/server/remote/bitbucket/bitbucket_test.go +++ b/server/remote/bitbucket/bitbucket_test.go @@ -126,12 +126,12 @@ func Test_bitbucket(t *testing.T) { g.Describe("When requesting a repository", func() { g.It("Should return the details", func() { - repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name) + repo, err := c.Repo(ctx, fakeUser, "", fakeRepo.Owner, fakeRepo.Name) g.Assert(err).IsNil() g.Assert(repo.FullName).Equal(fakeRepo.FullName) }) g.It("Should handle not found errors", func() { - _, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name) + _, err := c.Repo(ctx, fakeUser, "", fakeRepoNotFound.Owner, fakeRepoNotFound.Name) g.Assert(err).IsNotNil() }) }) diff --git a/server/remote/bitbucket/convert.go b/server/remote/bitbucket/convert.go index cc835155e..38adcb15a 100644 --- a/server/remote/bitbucket/convert.go +++ b/server/remote/bitbucket/convert.go @@ -49,6 +49,7 @@ func convertStatus(status model.StatusValue) string { // structure to the common Woodpecker repository structure. func convertRepo(from *internal.Repo) *model.Repo { repo := model.Repo{ + RemoteID: model.RemoteID(from.UUID), Clone: cloneLink(from), Owner: strings.Split(from.FullName, "/")[0], Name: strings.Split(from.FullName, "/")[1], diff --git a/server/remote/bitbucket/internal/types.go b/server/remote/bitbucket/internal/types.go index 9327b5c21..17c895690 100644 --- a/server/remote/bitbucket/internal/types.go +++ b/server/remote/bitbucket/internal/types.go @@ -89,6 +89,7 @@ type LinkClone struct { } type Repo struct { + UUID string `json:"uuid"` Owner Account `json:"owner"` Name string `json:"name"` FullName string `json:"full_name"` diff --git a/server/remote/bitbucketserver/bitbucketserver.go b/server/remote/bitbucketserver/bitbucketserver.go index 8a5d04617..7df224c08 100644 --- a/server/remote/bitbucketserver/bitbucketserver.go +++ b/server/remote/bitbucketserver/bitbucketserver.go @@ -151,7 +151,7 @@ func (*Config) TeamPerm(u *model.User, org string) (*model.Perm, error) { return nil, nil } -func (c *Config) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) { +func (c *Config) Repo(ctx context.Context, u *model.User, _ model.RemoteID, owner, name string) (*model.Repo, error) { repo, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).FindRepo(owner, name) if err != nil { return nil, err diff --git a/server/remote/bitbucketserver/convert.go b/server/remote/bitbucketserver/convert.go index 6cb40ff26..4978f196c 100644 --- a/server/remote/bitbucketserver/convert.go +++ b/server/remote/bitbucketserver/convert.go @@ -51,6 +51,7 @@ func convertStatus(status model.StatusValue) string { // structure to the common Woodpecker repository structure. func convertRepo(from *internal.Repo) *model.Repo { repo := model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(from.ID)), Name: from.Slug, Owner: from.Project.Key, Branch: "master", diff --git a/server/remote/bitbucketserver/parse.go b/server/remote/bitbucketserver/parse.go index 1940a050d..e5662fbd8 100644 --- a/server/remote/bitbucketserver/parse.go +++ b/server/remote/bitbucketserver/parse.go @@ -32,6 +32,7 @@ func parseHook(r *http.Request, baseURL string) (*model.Repo, *model.Build, erro } build := convertPushHook(hook, baseURL) repo := &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(hook.Repository.ID)), Name: hook.Repository.Slug, Owner: hook.Repository.Project.Key, FullName: fmt.Sprintf("%s/%s", hook.Repository.Project.Key, hook.Repository.Slug), diff --git a/server/remote/coding/coding.go b/server/remote/coding/coding.go index 70ab1403a..86e79eac4 100644 --- a/server/remote/coding/coding.go +++ b/server/remote/coding/coding.go @@ -160,8 +160,8 @@ func (c *Coding) TeamPerm(u *model.User, org string) (*model.Perm, error) { return nil, nil } -// Repo fetches the named repository from the remote system. -func (c *Coding) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) { +// Repo fetches the repository from the remote system. +func (c *Coding) Repo(ctx context.Context, u *model.User, _ model.RemoteID, owner, name string) (*model.Repo, error) { client := c.newClient(ctx, u) project, err := client.GetProject(owner, name) if err != nil { @@ -172,6 +172,7 @@ func (c *Coding) Repo(ctx context.Context, u *model.User, owner, name string) (* return nil, err } return &model.Repo{ + // TODO(1138) RemoteID: project.ID, Owner: project.Owner, Name: project.Name, FullName: projectFullName(project.Owner, project.Name), @@ -199,6 +200,7 @@ func (c *Coding) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error return nil, err } repo := &model.Repo{ + // TODO(1138) RemoteID: project.ID, Owner: project.Owner, Name: project.Name, FullName: projectFullName(project.Owner, project.Name), diff --git a/server/remote/coding/coding_test.go b/server/remote/coding/coding_test.go index 3296fe788..da27d9f18 100644 --- a/server/remote/coding/coding_test.go +++ b/server/remote/coding/coding_test.go @@ -108,7 +108,7 @@ func Test_coding(t *testing.T) { g.Describe("When requesting a repository", func() { g.It("Should return the details", func() { - repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name) + repo, err := c.Repo(ctx, fakeUser, "", fakeRepo.Owner, fakeRepo.Name) g.Assert(err).IsNil() g.Assert(repo.FullName).Equal(fakeRepo.FullName) g.Assert(repo.Avatar).Equal(s.URL + fakeRepo.Avatar) @@ -119,7 +119,7 @@ func Test_coding(t *testing.T) { g.Assert(repo.IsSCMPrivate).Equal(fakeRepo.IsSCMPrivate) }) g.It("Should handle not found errors", func() { - _, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name) + _, err := c.Repo(ctx, fakeUser, "", fakeRepoNotFound.Owner, fakeRepoNotFound.Name) g.Assert(err).IsNotNil() }) }) diff --git a/server/remote/coding/hook.go b/server/remote/coding/hook.go index ffd6c7be0..158025ac8 100644 --- a/server/remote/coding/hook.go +++ b/server/remote/coding/hook.go @@ -137,6 +137,7 @@ func convertRepository(repo *Repository) (*model.Repo, error) { } return &model.Repo{ + // TODO RemoteID: repo.ID, Owner: matches[1], Name: repo.Name, FullName: projectFullName(repo.Owner.GlobalKey, repo.Name), diff --git a/server/remote/coding/internal/coding.go b/server/remote/coding/internal/coding.go index a6c174082..5baacd91d 100644 --- a/server/remote/coding/internal/coding.go +++ b/server/remote/coding/internal/coding.go @@ -49,12 +49,12 @@ func NewClient(ctx context.Context, baseURL, apiPath, token, agent string, clien } } -// Generic GET for requesting Coding OAuth API +// Get Generic GET for requesting Coding OAuth API func (c *Client) Get(u string, params url.Values) ([]byte, error) { return c.Do(http.MethodGet, u, params) } -// Generic method for requesting Coding OAuth API +// Do Generic method for requesting Coding OAuth API func (c *Client) Do(method, u string, params url.Values) ([]byte, error) { if params == nil { params = url.Values{} diff --git a/server/remote/gitea/fixtures/handler.go b/server/remote/gitea/fixtures/handler.go index 399abdb00..606444856 100644 --- a/server/remote/gitea/fixtures/handler.go +++ b/server/remote/gitea/fixtures/handler.go @@ -27,6 +27,7 @@ func Handler() http.Handler { e := gin.New() e.GET("/api/v1/repos/:owner/:name", getRepo) + e.GET("/api/v1/repositories/:id", getRepoByID) e.GET("/api/v1/repos/:owner/:name/raw/:commit/:file", getRepoFile) e.POST("/api/v1/repos/:owner/:name/hooks", createRepoHook) e.GET("/api/v1/repos/:owner/:name/hooks", listRepoHooks) @@ -51,6 +52,15 @@ func getRepo(c *gin.Context) { } } +func getRepoByID(c *gin.Context) { + switch c.Param("id") { + case "repo_not_found": + c.String(404, "") + default: + c.String(200, repoPayload) + } +} + func createRepoCommitStatus(c *gin.Context) { if c.Param("commit") == "v1.0.0" || c.Param("commit") == "9ecad50" { c.String(200, repoPayload) @@ -124,6 +134,7 @@ const listRepoHookPayloads = ` const repoPayload = ` { + "id": 5, "owner": { "login": "test_name", "email": "octocat@github.com", @@ -146,6 +157,7 @@ const repoFilePayload = `{ platform: linux/amd64 }` const userRepoPayload = ` [ { + "id": 5, "owner": { "login": "test_name", "email": "octocat@github.com", diff --git a/server/remote/gitea/gitea.go b/server/remote/gitea/gitea.go index d72f08a72..193d23ee1 100644 --- a/server/remote/gitea/gitea.go +++ b/server/remote/gitea/gitea.go @@ -26,6 +26,7 @@ import ( "net/url" "path" "path/filepath" + "strconv" "strings" "time" @@ -225,13 +226,25 @@ func (c *Gitea) TeamPerm(u *model.User, org string) (*model.Perm, error) { return nil, nil } -// Repo returns the named Gitea repository. -func (c *Gitea) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) { +// Repo returns the Gitea repository. +func (c *Gitea) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) { client, err := c.newClientToken(ctx, u.Token) if err != nil { return nil, err } + if id.IsValid() { + intID, err := strconv.ParseInt(string(id), 10, 64) + if err != nil { + return nil, err + } + repo, _, err := client.GetRepoByID(intID) + if err != nil { + return nil, err + } + return toRepo(repo), nil + } + repo, _, err := client.GetRepo(owner, name) if err != nil { return nil, err diff --git a/server/remote/gitea/gitea_test.go b/server/remote/gitea/gitea_test.go index c2c3d7ae5..aa57cb6af 100644 --- a/server/remote/gitea/gitea_test.go +++ b/server/remote/gitea/gitea_test.go @@ -76,7 +76,7 @@ func Test_gitea(t *testing.T) { g.Describe("Requesting a repository", func() { g.It("Should return the repository details", func() { - repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name) + repo, err := c.Repo(ctx, fakeUser, fakeRepo.RemoteID, fakeRepo.Owner, fakeRepo.Name) g.Assert(err).IsNil() g.Assert(repo.Owner).Equal(fakeRepo.Owner) g.Assert(repo.Name).Equal(fakeRepo.Name) @@ -86,7 +86,7 @@ func Test_gitea(t *testing.T) { g.Assert(repo.Link).Equal("http://localhost/test_name/repo_name") }) g.It("Should handle a not found error", func() { - _, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name) + _, err := c.Repo(ctx, fakeUser, "0", fakeRepoNotFound.Owner, fakeRepoNotFound.Name) g.Assert(err).IsNotNil() }) }) @@ -109,6 +109,7 @@ func Test_gitea(t *testing.T) { g.It("Should return the repository list", func() { repos, err := c.Repos(ctx, fakeUser) g.Assert(err).IsNil() + g.Assert(repos[0].RemoteID).Equal(fakeRepo.RemoteID) g.Assert(repos[0].Owner).Equal(fakeRepo.Owner) g.Assert(repos[0].Name).Equal(fakeRepo.Name) g.Assert(repos[0].FullName).Equal(fakeRepo.Owner + "/" + fakeRepo.Name) @@ -168,6 +169,7 @@ var ( fakeRepo = &model.Repo{ Clone: "http://gitea.com/test_name/repo_name.git", + RemoteID: "5", Owner: "test_name", Name: "repo_name", FullName: "test_name/repo_name", diff --git a/server/remote/gitea/helper.go b/server/remote/gitea/helper.go index 3712a3a49..b0e18e32f 100644 --- a/server/remote/gitea/helper.go +++ b/server/remote/gitea/helper.go @@ -36,6 +36,7 @@ func toRepo(from *gitea.Repository) *model.Repo { from.Owner.AvatarURL, ) return &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(from.ID)), SCMKind: model.RepoGit, Name: name, Owner: from.Owner.UserName, @@ -185,6 +186,7 @@ func buildFromPullRequest(hook *pullRequestHook) *model.Build { // helper function that extracts the Repository data from a Gitea push hook func repoFromPush(hook *pushHook) *model.Repo { return &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)), Name: hook.Repo.Name, Owner: hook.Repo.Owner.Username, FullName: hook.Repo.FullName, @@ -195,6 +197,7 @@ func repoFromPush(hook *pushHook) *model.Repo { // helper function that extracts the Repository data from a Gitea pull_request hook func repoFromPullRequest(hook *pullRequestHook) *model.Repo { return &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)), Name: hook.Repo.Name, Owner: hook.Repo.Owner.Username, FullName: hook.Repo.FullName, diff --git a/server/remote/github/convert.go b/server/remote/github/convert.go index ee3f18af7..0d4aa90f0 100644 --- a/server/remote/github/convert.go +++ b/server/remote/github/convert.go @@ -15,6 +15,8 @@ package github import ( + "fmt" + "github.com/google/go-github/v39/github" "github.com/woodpecker-ci/woodpecker/server/model" @@ -82,6 +84,7 @@ func convertDesc(status model.StatusValue) string { // structure to the common Woodpecker repository structure. func convertRepo(from *github.Repository) *model.Repo { repo := &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(from.GetID())), Name: from.GetName(), FullName: from.GetFullName(), Link: from.GetHTMLURL(), @@ -142,6 +145,7 @@ func convertTeam(from *github.Organization) *model.Team { // from a webhook and convert to the common Woodpecker repository structure. func convertRepoHook(eventRepo *github.PushEventRepository) *model.Repo { repo := &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(eventRepo.GetID())), Owner: eventRepo.GetOwner().GetLogin(), Name: eventRepo.GetName(), FullName: eventRepo.GetFullName(), diff --git a/server/remote/github/fixtures/handler.go b/server/remote/github/fixtures/handler.go index d3e0c314d..320805a51 100644 --- a/server/remote/github/fixtures/handler.go +++ b/server/remote/github/fixtures/handler.go @@ -27,6 +27,7 @@ func Handler() http.Handler { e := gin.New() e.GET("/api/v3/repos/:owner/:name", getRepo) + e.GET("/api/v3/repositories/:id", getRepoByID) e.GET("/api/v3/orgs/:org/memberships/:user", getMembership) e.GET("/api/v3/user/memberships/orgs/:org", getMembership) @@ -42,6 +43,15 @@ func getRepo(c *gin.Context) { } } +func getRepoByID(c *gin.Context) { + switch c.Param("id") { + case "repo_not_found": + c.String(404, "") + default: + c.String(200, repoPayload) + } +} + func getMembership(c *gin.Context) { switch c.Param("org") { case "org_not_found": @@ -55,6 +65,7 @@ func getMembership(c *gin.Context) { var repoPayload = ` { + "id": 5, "owner": { "login": "octocat", "avatar_url": "https://github.com/images/error/octocat_happy.gif" diff --git a/server/remote/github/github.go b/server/remote/github/github.go index 122db8fd2..3c26582ec 100644 --- a/server/remote/github/github.go +++ b/server/remote/github/github.go @@ -163,9 +163,22 @@ func (c *client) Teams(ctx context.Context, u *model.User) ([]*model.Team, error return teams, nil } -// Repo returns the named GitHub repository. -func (c *client) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) { +// Repo returns the GitHub repository. +func (c *client) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) { client := c.newClientToken(ctx, u.Token) + + if id.IsValid() { + intID, err := strconv.ParseInt(string(id), 10, 64) + if err != nil { + return nil, err + } + repo, _, err := client.Repositories.GetByID(ctx, intID) + if err != nil { + return nil, err + } + return convertRepo(repo), nil + } + repo, _, err := client.Repositories.Get(ctx, owner, name) if err != nil { return nil, err @@ -531,7 +544,7 @@ func (c *client) loadChangedFilesFromPullRequest(ctx context.Context, pull *gith return build, nil } - repo, err := _store.GetRepoName(tmpRepo.Owner + "/" + tmpRepo.Name) + repo, err := _store.GetRepoNameFallback(tmpRepo.RemoteID, tmpRepo.FullName) if err != nil { return nil, err } diff --git a/server/remote/github/github_test.go b/server/remote/github/github_test.go index 4f8bf23c5..c3b72076a 100644 --- a/server/remote/github/github_test.go +++ b/server/remote/github/github_test.go @@ -77,8 +77,9 @@ func Test_github(t *testing.T) { g.Describe("Requesting a repository", func() { g.It("Should return the repository details", func() { - repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name) + repo, err := c.Repo(ctx, fakeUser, fakeRepo.RemoteID, fakeRepo.Owner, fakeRepo.Name) g.Assert(err).IsNil() + g.Assert(repo.RemoteID).Equal(fakeRepo.RemoteID) g.Assert(repo.Owner).Equal(fakeRepo.Owner) g.Assert(repo.Name).Equal(fakeRepo.Name) g.Assert(repo.FullName).Equal(fakeRepo.FullName) @@ -87,7 +88,7 @@ func Test_github(t *testing.T) { g.Assert(repo.Link).Equal(fakeRepo.Link) }) g.It("Should handle a not found error", func() { - _, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name) + _, err := c.Repo(ctx, fakeUser, "0", fakeRepoNotFound.Owner, fakeRepoNotFound.Name) g.Assert(err).IsNotNil() }) }) @@ -131,6 +132,7 @@ var ( } fakeRepo = &model.Repo{ + RemoteID: "5", Owner: "octocat", Name: "Hello-World", FullName: "octocat/Hello-World", diff --git a/server/remote/gitlab/convert.go b/server/remote/gitlab/convert.go index 1e14bc08b..c6123c3fd 100644 --- a/server/remote/gitlab/convert.go +++ b/server/remote/gitlab/convert.go @@ -37,6 +37,7 @@ func (g *Gitlab) convertGitlabRepo(_repo *gitlab.Project) (*model.Repo, error) { owner := strings.Join(parts[:len(parts)-1], "/") name := parts[len(parts)-1] repo := &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(_repo.ID)), Owner: owner, Name: name, FullName: _repo.PathWithNamespace, @@ -87,6 +88,7 @@ func convertMergeRequestHook(hook *gitlab.MergeEvent, req *http.Request) (int, * repo.FullName = fmt.Sprintf("%s/%s", repo.Owner, repo.Name) } + repo.RemoteID = model.RemoteID(fmt.Sprint(obj.TargetProjectID)) repo.Link = target.WebURL if target.GitHTTPURL != "" { @@ -140,6 +142,7 @@ func convertPushHook(hook *gitlab.PushEvent) (*model.Repo, *model.Build, error) return nil, nil, err } + repo.RemoteID = model.RemoteID(fmt.Sprint(hook.ProjectID)) repo.Avatar = hook.Project.AvatarURL repo.Link = hook.Project.WebURL repo.Clone = hook.Project.GitHTTPURL @@ -191,6 +194,7 @@ func convertTagHook(hook *gitlab.TagEvent) (*model.Repo, *model.Build, error) { return nil, nil, err } + repo.RemoteID = model.RemoteID(fmt.Sprint(hook.ProjectID)) repo.Avatar = hook.Project.AvatarURL repo.Link = hook.Project.WebURL repo.Clone = hook.Project.GitHTTPURL diff --git a/server/remote/gitlab/gitlab.go b/server/remote/gitlab/gitlab.go index aa8810a68..c086f9f9c 100644 --- a/server/remote/gitlab/gitlab.go +++ b/server/remote/gitlab/gitlab.go @@ -21,6 +21,7 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "time" @@ -225,13 +226,25 @@ func (g *Gitlab) getProject(ctx context.Context, client *gitlab.Client, owner, n return repo, nil } -// Repo fetches the named repository from the remote system. -func (g *Gitlab) Repo(ctx context.Context, user *model.User, owner, name string) (*model.Repo, error) { +// Repo fetches the repository from the remote system. +func (g *Gitlab) Repo(ctx context.Context, user *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) { client, err := newClient(g.URL, user.Token, g.SkipVerify) if err != nil { return nil, err } + if id.IsValid() { + intID, err := strconv.ParseInt(string(id), 10, 64) + if err != nil { + return nil, err + } + _repo, _, err := client.Projects.GetProject(int(intID), nil, gitlab.WithContext(ctx)) + if err != nil { + return nil, err + } + return g.convertGitlabRepo(_repo) + } + _repo, err := g.getProject(ctx, client, owner, name) if err != nil { return nil, err @@ -667,7 +680,7 @@ func (g *Gitlab) loadChangedFilesFromMergeRequest(ctx context.Context, tmpRepo * return build, nil } - repo, err := _store.GetRepoName(tmpRepo.Owner + "/" + tmpRepo.Name) + repo, err := _store.GetRepoNameFallback(tmpRepo.RemoteID, tmpRepo.FullName) if err != nil { return nil, err } diff --git a/server/remote/gitlab/gitlab_test.go b/server/remote/gitlab/gitlab_test.go index 8a3ed7381..7cc84050f 100644 --- a/server/remote/gitlab/gitlab_test.go +++ b/server/remote/gitlab/gitlab_test.go @@ -93,7 +93,7 @@ func Test_Gitlab(t *testing.T) { // Test repository method g.Describe("Repo", func() { g.It("Should return valid repo", func() { - _repo, err := client.Repo(ctx, &user, "diaspora", "diaspora-client") + _repo, err := client.Repo(ctx, &user, "0", "diaspora", "diaspora-client") assert.NoError(t, err) assert.Equal(t, "diaspora-client", _repo.Name) assert.Equal(t, "diaspora", _repo.Owner) @@ -101,7 +101,7 @@ func Test_Gitlab(t *testing.T) { }) g.It("Should return error, when repo not exist", func() { - _, err := client.Repo(ctx, &user, "not-existed", "not-existed") + _, err := client.Repo(ctx, &user, "0", "not-existed", "not-existed") assert.Error(t, err) }) }) diff --git a/server/remote/gogs/fixtures/handler.go b/server/remote/gogs/fixtures/handler.go index b67b78b21..0fdb7925c 100644 --- a/server/remote/gogs/fixtures/handler.go +++ b/server/remote/gogs/fixtures/handler.go @@ -83,6 +83,7 @@ func getUserRepos(c *gin.Context) { const repoPayload = ` { + "id": 5, "owner": { "username": "test_name", "email": "octocat@github.com", @@ -105,6 +106,7 @@ const repoFilePayload = `{ platform: linux/amd64 }` const userRepoPayload = ` [ { + "id": 5, "owner": { "username": "test_name", "email": "octocat@github.com", diff --git a/server/remote/gogs/gogs.go b/server/remote/gogs/gogs.go index 4a69a0ec6..56f68e0ff 100644 --- a/server/remote/gogs/gogs.go +++ b/server/remote/gogs/gogs.go @@ -148,7 +148,7 @@ func (c *client) Teams(ctx context.Context, u *model.User) ([]*model.Team, error } // Repo returns the named Gogs repository. -func (c *client) Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) { +func (c *client) Repo(ctx context.Context, u *model.User, _ model.RemoteID, owner, name string) (*model.Repo, error) { client := c.newClientToken(u.Token) repo, err := client.GetRepo(owner, name) if err != nil { @@ -160,7 +160,7 @@ func (c *client) Repo(ctx context.Context, u *model.User, owner, name string) (* // Repos returns a list of all repositories for the Gogs account, including // organization repositories. func (c *client) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) { - repos := []*model.Repo{} + var repos []*model.Repo client := c.newClientToken(u.Token) all, err := client.ListMyRepos() diff --git a/server/remote/gogs/gogs_test.go b/server/remote/gogs/gogs_test.go index 52874ee50..e922853bb 100644 --- a/server/remote/gogs/gogs_test.go +++ b/server/remote/gogs/gogs_test.go @@ -85,7 +85,7 @@ func Test_gogs(t *testing.T) { g.Describe("Requesting a repository", func() { g.It("Should return the repository details", func() { - repo, err := c.Repo(ctx, fakeUser, fakeRepo.Owner, fakeRepo.Name) + repo, err := c.Repo(ctx, fakeUser, fakeRepo.RemoteID, fakeRepo.Owner, fakeRepo.Name) g.Assert(err).IsNil() g.Assert(repo.Owner).Equal(fakeRepo.Owner) g.Assert(repo.Name).Equal(fakeRepo.Name) @@ -95,7 +95,7 @@ func Test_gogs(t *testing.T) { g.Assert(repo.Link).Equal("http://localhost/test_name/repo_name") }) g.It("Should handle a not found error", func() { - _, err := c.Repo(ctx, fakeUser, fakeRepoNotFound.Owner, fakeRepoNotFound.Name) + _, err := c.Repo(ctx, fakeUser, "0", fakeRepoNotFound.Owner, fakeRepoNotFound.Name) g.Assert(err).IsNotNil() }) }) @@ -118,6 +118,7 @@ func Test_gogs(t *testing.T) { g.It("Should return the repository list", func() { repos, err := c.Repos(ctx, fakeUser) g.Assert(err).IsNil() + g.Assert(repos[0].RemoteID).Equal(fakeRepo.RemoteID) g.Assert(repos[0].Owner).Equal(fakeRepo.Owner) g.Assert(repos[0].Name).Equal(fakeRepo.Name) g.Assert(repos[0].FullName).Equal(fakeRepo.Owner + "/" + fakeRepo.Name) @@ -181,6 +182,7 @@ var ( } fakeRepo = &model.Repo{ + RemoteID: "5", Clone: "http://gogs.com/test_name/repo_name.git", Owner: "test_name", Name: "repo_name", diff --git a/server/remote/gogs/helper.go b/server/remote/gogs/helper.go index d711ae75b..3656f8cd5 100644 --- a/server/remote/gogs/helper.go +++ b/server/remote/gogs/helper.go @@ -39,6 +39,7 @@ func toRepo(from *gogs.Repository, privateMode bool) *model.Repo { private = true } return &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(from.ID)), SCMKind: model.RepoGit, Name: name, Owner: from.Owner.UserName, @@ -159,6 +160,7 @@ func buildFromPullRequest(hook *pullRequestHook) *model.Build { // helper function that extracts the Repository data from a Gogs push hook func repoFromPush(hook *pushHook) *model.Repo { return &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)), Name: hook.Repo.Name, Owner: hook.Repo.Owner.Username, FullName: hook.Repo.FullName, @@ -169,6 +171,7 @@ func repoFromPush(hook *pushHook) *model.Repo { // helper function that extracts the Repository data from a Gogs pull_request hook func repoFromPullRequest(hook *pullRequestHook) *model.Repo { return &model.Repo{ + RemoteID: model.RemoteID(fmt.Sprint(hook.Repo.ID)), Name: hook.Repo.Name, Owner: hook.Repo.Owner.Username, FullName: hook.Repo.FullName, diff --git a/server/remote/mocks/remote.go b/server/remote/mocks/remote.go index 3b2629f80..c7f4104b7 100644 --- a/server/remote/mocks/remote.go +++ b/server/remote/mocks/remote.go @@ -295,13 +295,13 @@ func (_m *Remote) Perm(ctx context.Context, u *model.User, r *model.Repo) (*mode return r0, r1 } -// Repo provides a mock function with given fields: ctx, u, owner, name -func (_m *Remote) Repo(ctx context.Context, u *model.User, owner string, name string) (*model.Repo, error) { - ret := _m.Called(ctx, u, owner, name) +// Repo provides a mock function with given fields: ctx, u, id, owner, name +func (_m *Remote) Repo(ctx context.Context, u *model.User, id model.RemoteID, owner string, name string) (*model.Repo, error) { + ret := _m.Called(ctx, u, id, owner, name) var r0 *model.Repo - if rf, ok := ret.Get(0).(func(context.Context, *model.User, string, string) *model.Repo); ok { - r0 = rf(ctx, u, owner, name) + if rf, ok := ret.Get(0).(func(context.Context, *model.User, model.RemoteID, string, string) *model.Repo); ok { + r0 = rf(ctx, u, id, owner, name) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*model.Repo) @@ -309,8 +309,8 @@ func (_m *Remote) Repo(ctx context.Context, u *model.User, owner string, name st } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, *model.User, string, string) error); ok { - r1 = rf(ctx, u, owner, name) + if rf, ok := ret.Get(1).(func(context.Context, *model.User, model.RemoteID, string, string) error); ok { + r1 = rf(ctx, u, id, owner, name) } else { r1 = ret.Error(1) } diff --git a/server/remote/remote.go b/server/remote/remote.go index 7599c59ed..0141da83e 100644 --- a/server/remote/remote.go +++ b/server/remote/remote.go @@ -42,8 +42,8 @@ type Remote interface { // Teams fetches a list of team memberships from the remote system. Teams(ctx context.Context, u *model.User) ([]*model.Team, error) - // Repo fetches the named repository from the remote system. - Repo(ctx context.Context, u *model.User, owner, name string) (*model.Repo, error) + // Repo fetches the repository from the remote system, preferred is using the ID, fallback is owner/name. + Repo(ctx context.Context, u *model.User, id model.RemoteID, owner, name string) (*model.Repo, error) // Repos fetches a list of repos from the remote system. Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) diff --git a/server/router/middleware/session/repo.go b/server/router/middleware/session/repo.go index 773d1baec..b8cded22a 100644 --- a/server/router/middleware/session/repo.go +++ b/server/router/middleware/session/repo.go @@ -100,7 +100,7 @@ func SetPerm() gin.HandlerFunc { perm, err = server.Config.Services.Remote.Perm(c, user, repo) if err == nil { log.Debug().Msgf("Synced user permission for %s %s", user.Login, repo.FullName) - perm.Repo = repo.FullName + perm.Repo = repo perm.UserID = user.ID perm.Synced = time.Now().Unix() if err := _store.PermUpsert(perm); err != nil { diff --git a/server/shared/userSyncer.go b/server/shared/userSyncer.go index f268de43c..1a0c20b05 100644 --- a/server/shared/userSyncer.go +++ b/server/shared/userSyncer.go @@ -74,7 +74,7 @@ func (s *Syncer) Sync(ctx context.Context, user *model.User, flatPermissions boo repo.Perm = &model.Perm{ UserID: user.ID, RepoID: repo.ID, - Repo: repo.FullName, + Repo: repo, Synced: unix, } diff --git a/server/store/datastore/feed_test.go b/server/store/datastore/feed_test.go index 4555e945e..878a566b1 100644 --- a/server/store/datastore/feed_test.go +++ b/server/store/datastore/feed_test.go @@ -37,18 +37,21 @@ func TestRepoListLatest(t *testing.T) { Owner: "bradrydzewski", Name: "test", FullName: "bradrydzewski/test", + RemoteID: "1", IsActive: true, } repo2 := &model.Repo{ Owner: "test", Name: "test", FullName: "test/test", + RemoteID: "2", IsActive: true, } repo3 := &model.Repo{ Owner: "octocat", Name: "hello-world", FullName: "octocat/hello-world", + RemoteID: "3", IsActive: true, } assert.NoError(t, store.CreateRepo(repo1)) @@ -56,8 +59,8 @@ func TestRepoListLatest(t *testing.T) { assert.NoError(t, store.CreateRepo(repo3)) for _, perm := range []*model.Perm{ - {UserID: user.ID, Repo: repo1.FullName, Push: true, Admin: false}, - {UserID: user.ID, Repo: repo2.FullName, Push: true, Admin: true}, + {UserID: user.ID, Repo: repo1, Push: true, Admin: false}, + {UserID: user.ID, Repo: repo2, Push: true, Admin: true}, } { assert.NoError(t, store.PermUpsert(perm)) } diff --git a/server/store/datastore/migration/migration.go b/server/store/datastore/migration/migration.go index 4cd0f9aae..95a61c026 100644 --- a/server/store/datastore/migration/migration.go +++ b/server/store/datastore/migration/migration.go @@ -53,6 +53,7 @@ var allBeans = []interface{}{ new(model.User), new(model.ServerConfig), new(model.Cron), + new(model.Redirection), } type migrations struct { diff --git a/server/store/datastore/permission.go b/server/store/datastore/permission.go index 832c9d26b..5b97fc327 100644 --- a/server/store/datastore/permission.go +++ b/server/store/datastore/permission.go @@ -45,13 +45,13 @@ func (s storage) PermUpsert(perm *model.Perm) error { } func (s storage) permUpsert(sess *xorm.Session, perm *model.Perm) error { - if perm.RepoID == 0 && len(perm.Repo) == 0 { + if perm.RepoID == 0 && perm.Repo == nil { return fmt.Errorf("could not determine repo for permission: %v", perm) } - // lookup repo based on name if possible - if perm.RepoID == 0 && len(perm.Repo) != 0 { - r, err := s.getRepoName(sess, perm.Repo) + // lookup repo based on name or remote ID if possible + if perm.RepoID == 0 && perm.Repo != nil { + r, err := s.getRepoNameFallback(sess, perm.Repo.RemoteID, perm.Repo.FullName) if err != nil { return err } diff --git a/server/store/datastore/permission_test.go b/server/store/datastore/permission_test.go index 92dda7708..79a2d16d8 100644 --- a/server/store/datastore/permission_test.go +++ b/server/store/datastore/permission_test.go @@ -32,6 +32,7 @@ func TestPermFind(t *testing.T) { FullName: "bradrydzewski/test", Owner: "bradrydzewski", Name: "test", + RemoteID: "1", } assert.NoError(t, store.CreateRepo(repo)) @@ -39,7 +40,7 @@ func TestPermFind(t *testing.T) { &model.Perm{ UserID: user.ID, RepoID: repo.ID, - Repo: repo.FullName, + Repo: repo, Pull: true, Push: false, Admin: false, @@ -76,6 +77,7 @@ func TestPermUpsert(t *testing.T) { FullName: "bradrydzewski/test", Owner: "bradrydzewski", Name: "test", + RemoteID: "1", } assert.NoError(t, store.CreateRepo(repo)) @@ -83,7 +85,7 @@ func TestPermUpsert(t *testing.T) { &model.Perm{ UserID: user.ID, RepoID: repo.ID, - Repo: repo.FullName, + Repo: repo, Pull: true, Push: false, Admin: false, @@ -118,7 +120,7 @@ func TestPermUpsert(t *testing.T) { &model.Perm{ UserID: user.ID, RepoID: repo.ID, - Repo: repo.FullName, + Repo: repo, Pull: true, Push: true, Admin: true, @@ -155,6 +157,7 @@ func TestPermDelete(t *testing.T) { FullName: "bradrydzewski/test", Owner: "bradrydzewski", Name: "test", + RemoteID: "1", } assert.NoError(t, store.CreateRepo(repo)) @@ -162,7 +165,7 @@ func TestPermDelete(t *testing.T) { &model.Perm{ UserID: user.ID, RepoID: repo.ID, - Repo: repo.FullName, + Repo: repo, Pull: true, Push: false, Admin: false, diff --git a/server/store/datastore/redirection.go b/server/store/datastore/redirection.go new file mode 100644 index 000000000..4f6cf8c05 --- /dev/null +++ b/server/store/datastore/redirection.go @@ -0,0 +1,47 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "github.com/woodpecker-ci/woodpecker/server/model" + "xorm.io/xorm" +) + +func (s storage) GetRedirection(fullName string) (*model.Redirection, error) { + sess := s.engine.NewSession() + defer sess.Close() + return s.getRedirection(sess, fullName) +} + +func (s storage) getRedirection(e *xorm.Session, fullName string) (*model.Redirection, error) { + repo := new(model.Redirection) + return repo, wrapGet(e.Where("repo_full_name = ?", fullName).Get(repo)) +} + +func (s storage) CreateRedirection(redirect *model.Redirection) error { + sess := s.engine.NewSession() + defer sess.Close() + return s.createRedirection(sess, redirect) +} + +func (s storage) createRedirection(e *xorm.Session, redirect *model.Redirection) error { + // only Insert set auto created ID back to object + _, err := e.Insert(redirect) + return err +} + +func (s storage) HasRedirectionForRepo(repoID int64, fullName string) (bool, error) { + return s.engine.Where("repo_id = ? ", repoID).And("repo_full_name = ?", fullName).Get(new(model.Redirection)) +} diff --git a/server/store/datastore/redirection_test.go b/server/store/datastore/redirection_test.go new file mode 100644 index 000000000..e536bc579 --- /dev/null +++ b/server/store/datastore/redirection_test.go @@ -0,0 +1,84 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/woodpecker-ci/woodpecker/server/model" +) + +func TestGetRedirection(t *testing.T) { + store, closer := newTestStore(t, new(model.Redirection)) + defer closer() + + redirection := &model.Redirection{ + RepoID: 1, + FullName: "foo/bar", + } + if err := store.CreateRedirection(redirection); err != nil { + t.Errorf("Unexpected error: insert redirection: %s", err) + return + } + redirectionFromStore, err := store.GetRedirection("foo/bar") + assert.NoError(t, err) + assert.NotNil(t, redirectionFromStore) + assert.Equal(t, redirection.RepoID, redirectionFromStore.RepoID) + _, err = store.GetRedirection("foo/baz") + assert.ErrorIs(t, err, RecordNotExist) +} + +func TestCreateRedirection(t *testing.T) { + store, closer := newTestStore(t, new(model.Redirection)) + defer closer() + + redirection := &model.Redirection{ + RepoID: 1, + FullName: "foo/bar", + } + assert.NoError(t, store.CreateRedirection(redirection)) +} + +func TestHasRedirectionForRepo(t *testing.T) { + store, closer := newTestStore(t, new(model.Redirection)) + defer closer() + + redirection := &model.Redirection{ + RepoID: 1, + FullName: "foo/bar", + } + if err := store.CreateRedirection(redirection); err != nil { + t.Errorf("Unexpected error: insert redirection: %s", err) + return + } + has, err := store.HasRedirectionForRepo(1, "foo/bar") + if err != nil { + t.Error(err) + return + } + if !has { + t.Errorf("Expected a redirection for %s", redirection.FullName) + } + has, err = store.HasRedirectionForRepo(1, "foo/baz") + if err != nil { + t.Error(err) + return + } + if has { + t.Errorf("Expected not finding a redirection for %s", redirection.FullName) + } +} diff --git a/server/store/datastore/repo.go b/server/store/datastore/repo.go index 2b3794170..a46a174d9 100644 --- a/server/store/datastore/repo.go +++ b/server/store/datastore/repo.go @@ -27,10 +27,44 @@ func (s storage) GetRepo(id int64) (*model.Repo, error) { return repo, wrapGet(s.engine.ID(id).Get(repo)) } +func (s storage) GetRepoRemoteID(id model.RemoteID) (*model.Repo, error) { + sess := s.engine.NewSession() + defer sess.Close() + return s.getRepoRemoteID(sess, id) +} + +func (s storage) getRepoRemoteID(e *xorm.Session, id model.RemoteID) (*model.Repo, error) { + repo := new(model.Repo) + return repo, wrapGet(e.Where("remote_id = ?", id).Get(repo)) +} + +func (s storage) GetRepoNameFallback(remoteID model.RemoteID, fullName string) (*model.Repo, error) { + sess := s.engine.NewSession() + defer sess.Close() + return s.getRepoNameFallback(sess, remoteID, fullName) +} + +func (s storage) getRepoNameFallback(e *xorm.Session, remoteID model.RemoteID, fullName string) (*model.Repo, error) { + repo, err := s.getRepoRemoteID(e, remoteID) + if err == RecordNotExist { + return s.getRepoName(e, fullName) + } + return repo, err +} + func (s storage) GetRepoName(fullName string) (*model.Repo, error) { sess := s.engine.NewSession() defer sess.Close() - return s.getRepoName(sess, fullName) + repo, err := s.getRepoName(sess, fullName) + if err == RecordNotExist { + // the repository does not exist, so look for a redirection + redirect, err := s.getRedirection(sess, fullName) + if err != nil { + return nil, err + } + return s.GetRepo(redirect.RepoID) + } + return repo, err } func (s storage) getRepoName(e *xorm.Session, fullName string) (*model.Repo, error) { @@ -73,6 +107,9 @@ func (s storage) DeleteRepo(repo *model.Repo) error { if _, err := sess.Where("secret_repo_id = ?", repo.ID).Delete(new(model.Secret)); err != nil { return err } + if _, err := sess.Where("repo_id = ?", repo.ID).Delete(new(model.Redirection)); err != nil { + return err + } // delete related builds for startBuilds := 0; ; startBuilds += batchSize { @@ -128,23 +165,39 @@ func (s storage) RepoBatch(repos []*model.Repo) error { continue } - exist, err := sess. - Where("repo_owner = ? AND repo_name = ?", repos[i].Owner, repos[i].Name). - Exist(new(model.Repo)) - if err != nil { + repo, err := s.getRepoNameFallback(sess, repos[i].RemoteID, repos[i].FullName) + if err != nil && err != RecordNotExist { return err } + exist := err == nil // if there's an error, it must be a RecordNotExist if exist { - if _, err := sess. - Where("repo_owner = ? AND repo_name = ?", repos[i].Owner, repos[i].Name). - Cols("repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch"). - Update(repos[i]); err != nil { - return err + if repos[i].FullName != repo.FullName { + // create redirection + err := s.createRedirection(sess, &model.Redirection{RepoID: repo.ID, FullName: repo.FullName}) + if err != nil { + return err + } + } + if repos[i].RemoteID.IsValid() { + if _, err := sess. + Where("remote_id = ?", repos[i].RemoteID). + Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "remote_id"). + Update(repos[i]); err != nil { + return err + } + } else { + if _, err := sess. + Where("repo_owner = ?", repos[i].Owner). + And(" repo_name = ?", repos[i].Name). + Cols("repo_owner", "repo_name", "repo_full_name", "repo_scm", "repo_avatar", "repo_link", "repo_private", "repo_clone", "repo_branch", "remote_id"). + Update(repos[i]); err != nil { + return err + } } _, err := sess. - Where("repo_owner = ? AND repo_name = ?", repos[i].Owner, repos[i].Name). + Where("remote_id = ?", repos[i].RemoteID). Get(repos[i]) if err != nil { return err @@ -158,7 +211,7 @@ func (s storage) RepoBatch(repos []*model.Repo) error { if repos[i].Perm != nil { repos[i].Perm.RepoID = repos[i].ID - repos[i].Perm.Repo = repos[i].FullName + repos[i].Perm.Repo = repos[i] if err := s.permUpsert(sess, repos[i].Perm); err != nil { return err } diff --git a/server/store/datastore/repo_test.go b/server/store/datastore/repo_test.go index 92a2849ac..ff3ec1986 100644 --- a/server/store/datastore/repo_test.go +++ b/server/store/datastore/repo_test.go @@ -138,24 +138,27 @@ func TestRepoList(t *testing.T) { Owner: "bradrydzewski", Name: "test", FullName: "bradrydzewski/test", + RemoteID: "1", } repo2 := &model.Repo{ Owner: "test", Name: "test", FullName: "test/test", + RemoteID: "2", } repo3 := &model.Repo{ Owner: "octocat", Name: "hello-world", FullName: "octocat/hello-world", + RemoteID: "3", } assert.NoError(t, store.CreateRepo(repo1)) assert.NoError(t, store.CreateRepo(repo2)) assert.NoError(t, store.CreateRepo(repo3)) for _, perm := range []*model.Perm{ - {UserID: user.ID, Repo: repo1.FullName}, - {UserID: user.ID, Repo: repo2.FullName}, + {UserID: user.ID, Repo: repo1}, + {UserID: user.ID, Repo: repo2}, } { assert.NoError(t, store.PermUpsert(perm)) } @@ -191,21 +194,25 @@ func TestOwnedRepoList(t *testing.T) { Owner: "bradrydzewski", Name: "test", FullName: "bradrydzewski/test", + RemoteID: "1", } repo2 := &model.Repo{ Owner: "test", Name: "test", FullName: "test/test", + RemoteID: "2", } repo3 := &model.Repo{ Owner: "octocat", Name: "hello-world", FullName: "octocat/hello-world", + RemoteID: "3", } repo4 := &model.Repo{ Owner: "demo", Name: "demo", FullName: "demo/demo", + RemoteID: "4", } assert.NoError(t, store.CreateRepo(repo1)) assert.NoError(t, store.CreateRepo(repo2)) @@ -213,10 +220,10 @@ func TestOwnedRepoList(t *testing.T) { assert.NoError(t, store.CreateRepo(repo4)) for _, perm := range []*model.Perm{ - {UserID: user.ID, Repo: repo1.FullName, Push: true, Admin: false}, - {UserID: user.ID, Repo: repo2.FullName, Push: false, Admin: true}, - {UserID: user.ID, Repo: repo3.FullName}, - {UserID: user.ID, Repo: repo4.FullName}, + {UserID: user.ID, Repo: repo1, Push: true, Admin: false}, + {UserID: user.ID, Repo: repo2, Push: false, Admin: true}, + {UserID: user.ID, Repo: repo3}, + {UserID: user.ID, Repo: repo4}, } { assert.NoError(t, store.PermUpsert(perm)) } @@ -270,10 +277,11 @@ func TestRepoCount(t *testing.T) { } func TestRepoBatch(t *testing.T) { - store, closer := newTestStore(t, new(model.Repo), new(model.User), new(model.Perm)) + store, closer := newTestStore(t, new(model.Repo), new(model.User), new(model.Perm), new(model.Redirection)) defer closer() if !assert.NoError(t, store.CreateRepo(&model.Repo{ + RemoteID: "5", UserID: 1, FullName: "foo/bar", Owner: "foo", @@ -285,6 +293,7 @@ func TestRepoBatch(t *testing.T) { repos := []*model.Repo{ { + RemoteID: "5", UserID: 1, FullName: "foo/bar", Owner: "foo", @@ -299,6 +308,7 @@ func TestRepoBatch(t *testing.T) { }, }, { + RemoteID: "6", UserID: 1, FullName: "bar/baz", Owner: "bar", @@ -306,6 +316,7 @@ func TestRepoBatch(t *testing.T) { IsActive: true, }, { + RemoteID: "7", UserID: 1, FullName: "baz/qux", Owner: "baz", @@ -313,6 +324,7 @@ func TestRepoBatch(t *testing.T) { IsActive: true, }, { + RemoteID: "8", UserID: 0, // not activated repos do hot have a user id assigned FullName: "baz/notes", Owner: "baz", @@ -330,6 +342,7 @@ func TestRepoBatch(t *testing.T) { assert.True(t, perm.Admin) repo := &model.Repo{ + RemoteID: "5", FullName: "foo/bar", Owner: "foo", Name: "bar", @@ -373,7 +386,8 @@ func TestRepoCrud(t *testing.T) { new(model.File), new(model.Secret), new(model.Registry), - new(model.Config)) + new(model.Config), + new(model.Redirection)) defer closer() repo := model.Repo{ @@ -420,3 +434,46 @@ func TestRepoCrud(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, 1, buildCount) } + +func TestRepoRedirection(t *testing.T) { + store, closer := newTestStore(t, + new(model.Repo), + new(model.Redirection)) + defer closer() + + repo := model.Repo{ + UserID: 1, + RemoteID: "1", + FullName: "bradrydzewski/test", + Owner: "bradrydzewski", + Name: "test", + } + assert.NoError(t, store.CreateRepo(&repo)) + + repoUpdated := model.Repo{ + RemoteID: "1", + FullName: "bradrydzewski/test-renamed", + Owner: "bradrydzewski", + Name: "test-renamed", + } + + assert.NoError(t, store.RepoBatch([]*model.Repo{&repoUpdated})) + + // test redirection from old repo name + repoFromStore, err := store.GetRepoNameFallback("1", "bradrydzewski/test") + assert.NoError(t, err) + assert.Equal(t, repoFromStore.FullName, repoUpdated.FullName) + + // test getting repo without remote ID (use name fallback) + repo = model.Repo{ + UserID: 1, + FullName: "bradrydzewski/test-no-remote-id", + Owner: "bradrydzewski", + Name: "test-no-remote-id", + } + assert.NoError(t, store.CreateRepo(&repo)) + + repoFromStore, err = store.GetRepoNameFallback("", "bradrydzewski/test-no-remote-id") + assert.NoError(t, err) + assert.Equal(t, repoFromStore.FullName, repo.FullName) +} diff --git a/server/store/datastore/users_test.go b/server/store/datastore/users_test.go index 67c407286..93443f673 100644 --- a/server/store/datastore/users_test.go +++ b/server/store/datastore/users_test.go @@ -195,26 +195,29 @@ func TestUsers(t *testing.T) { Name: "test", FullName: "bradrydzewski/test", IsActive: true, + RemoteID: "1", } repo2 := &model.Repo{ Owner: "test", Name: "test", FullName: "test/test", IsActive: true, + RemoteID: "2", } repo3 := &model.Repo{ Owner: "octocat", Name: "hello-world", FullName: "octocat/hello-world", IsActive: true, + RemoteID: "3", } g.Assert(store.CreateRepo(repo1)).IsNil() g.Assert(store.CreateRepo(repo2)).IsNil() g.Assert(store.CreateRepo(repo3)).IsNil() for _, perm := range []*model.Perm{ - {UserID: user.ID, Repo: repo1.FullName, Push: true, Admin: false}, - {UserID: user.ID, Repo: repo2.FullName, Push: false, Admin: true}, + {UserID: user.ID, Repo: repo1, Push: true, Admin: false}, + {UserID: user.ID, Repo: repo2, Push: false, Admin: true}, } { g.Assert(store.PermUpsert(perm)).IsNil() } diff --git a/server/store/mocks/store.go b/server/store/mocks/store.go index 8f4ba4f8b..d9aa4bfc2 100644 --- a/server/store/mocks/store.go +++ b/server/store/mocks/store.go @@ -144,6 +144,20 @@ func (_m *Store) CreateBuild(_a0 *model.Build, _a1 ...*model.Proc) error { return r0 } +// CreateRedirection provides a mock function with given fields: redirection +func (_m *Store) CreateRedirection(redirection *model.Redirection) error { + ret := _m.Called(redirection) + + var r0 error + if rf, ok := ret.Get(0).(func(*model.Redirection) error); ok { + r0 = rf(redirection) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // CreateRepo provides a mock function with given fields: _a0 func (_m *Store) CreateRepo(_a0 *model.Repo) error { ret := _m.Called(_a0) @@ -643,6 +657,29 @@ func (_m *Store) GetBuildRef(_a0 *model.Repo, _a1 string) (*model.Build, error) return r0, r1 } +// GetRedirection provides a mock function with given fields: _a0 +func (_m *Store) GetRedirection(_a0 string) (*model.Redirection, error) { + ret := _m.Called(_a0) + + var r0 *model.Redirection + if rf, ok := ret.Get(0).(func(string) *model.Redirection); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Redirection) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetRepo provides a mock function with given fields: _a0 func (_m *Store) GetRepo(_a0 int64) (*model.Repo, error) { ret := _m.Called(_a0) @@ -710,6 +747,52 @@ func (_m *Store) GetRepoName(_a0 string) (*model.Repo, error) { return r0, r1 } +// GetRepoNameFallback provides a mock function with given fields: remoteID, fullName +func (_m *Store) GetRepoNameFallback(remoteID model.RemoteID, fullName string) (*model.Repo, error) { + ret := _m.Called(remoteID, fullName) + + var r0 *model.Repo + if rf, ok := ret.Get(0).(func(model.RemoteID, string) *model.Repo); ok { + r0 = rf(remoteID, fullName) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Repo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(model.RemoteID, string) error); ok { + r1 = rf(remoteID, fullName) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetRepoRemoteID provides a mock function with given fields: _a0 +func (_m *Store) GetRepoRemoteID(_a0 model.RemoteID) (*model.Repo, error) { + ret := _m.Called(_a0) + + var r0 *model.Repo + if rf, ok := ret.Get(0).(func(model.RemoteID) *model.Repo); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Repo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(model.RemoteID) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetUser provides a mock function with given fields: _a0 func (_m *Store) GetUser(_a0 int64) (*model.User, error) { ret := _m.Called(_a0) @@ -846,6 +929,27 @@ func (_m *Store) GlobalSecretList() ([]*model.Secret, error) { return r0, r1 } +// HasRedirectionForRepo provides a mock function with given fields: _a0, _a1 +func (_m *Store) HasRedirectionForRepo(_a0 int64, _a1 string) (bool, error) { + ret := _m.Called(_a0, _a1) + + var r0 bool + if rf, ok := ret.Get(0).(func(int64, string) bool); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64, string) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // LogFind provides a mock function with given fields: _a0 func (_m *Store) LogFind(_a0 *model.Proc) (io.ReadCloser, error) { ret := _m.Called(_a0) diff --git a/server/store/store.go b/server/store/store.go index b2a666f3d..1d6adf9be 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -46,6 +46,10 @@ type Store interface { // Repos // GetRepo gets a repo by unique ID. GetRepo(int64) (*model.Repo, error) + // GetRepoRemoteID gets a repo by its remote ID. + GetRepoRemoteID(model.RemoteID) (*model.Repo, error) + // GetRepoNameFallback gets the repo by its remote ID and if this doesn't exist by its full name. + GetRepoNameFallback(remoteID model.RemoteID, fullName string) (*model.Repo, error) // GetRepoName gets a repo by its full name. GetRepoName(string) (*model.Repo, error) // GetRepoCount gets a count of all repositories in the system. @@ -57,6 +61,14 @@ type Store interface { // DeleteRepo deletes a user repository. DeleteRepo(*model.Repo) error + // Redirections + // GetRedirection returns the redirection for the given full repo name + GetRedirection(string) (*model.Redirection, error) + // CreateRedirection creates a redirection + CreateRedirection(redirection *model.Redirection) error + // HasRedirectionForRepo checks if there's a redirection for the given repo and full name + HasRedirectionForRepo(int64, string) (bool, error) + // Builds // GetBuild gets a build by unique ID. GetBuild(int64) (*model.Build, error) @@ -87,7 +99,7 @@ type Store interface { // Feeds UserFeed(*model.User) ([]*model.Feed, error) - // Repositorys + // Repositories // RepoList TODO: paginate RepoList(user *model.User, owned bool) ([]*model.Repo, error) RepoListLatest(*model.User) ([]*model.Feed, error) diff --git a/server/swagger/doc.go b/server/swagger/doc.go index 56d093fd2..65010b6b1 100644 --- a/server/swagger/doc.go +++ b/server/swagger/doc.go @@ -14,15 +14,15 @@ // Package classification Drone API. // -// Schemes: http, https -// BasePath: /api -// Version: 1.0.0 +// Schemes: http, https +// BasePath: /api +// Version: 1.0.0 // -// Consumes: -// - application/json +// Consumes: +// - application/json // -// Produces: -// - application/json +// Produces: +// - application/json // // swagger:meta package swagger diff --git a/server/swagger/swagger.go b/server/swagger/swagger.go index 345f087b2..68efbf7af 100644 --- a/server/swagger/swagger.go +++ b/server/swagger/swagger.go @@ -24,54 +24,48 @@ import ( // // Get the user with the matching login. // -// Responses: -// 200: user -// +// Responses: +// 200: user func userFind(w http.ResponseWriter, r *http.Request) {} // swagger:route GET /user user getCurrentUser // // Get the currently authenticated user. // -// Responses: -// 200: user -// +// Responses: +// 200: user func userCurrent(w http.ResponseWriter, r *http.Request) {} // swagger:route GET /users user getUserList // // Get the list of all registered users. // -// Responses: -// 200: user -// +// Responses: +// 200: user func userList(w http.ResponseWriter, r *http.Request) {} // swagger:route GET /user/feed user getUserFeed // // Get the currently authenticated user's build feed. // -// Responses: -// 200: feed -// +// Responses: +// 200: feed func userFeed(w http.ResponseWriter, r *http.Request) {} // swagger:route DELETE /users/{login} user deleteUserLogin // // Delete the user with the matching login. // -// Responses: -// 200: user -// +// Responses: +// 200: user func userDelete(w http.ResponseWriter, r *http.Request) {} // swagger:route GET /user/repos user getUserRepos // // Get the currently authenticated user's active repository list. // -// Responses: -// 200: repos -// +// Responses: +// 200: repos func repoList(w http.ResponseWriter, r *http.Request) {} // swagger:response user diff --git a/web/src/store/repos.ts b/web/src/store/repos.ts index 9c9f194bf..b975c6b48 100644 --- a/web/src/store/repos.ts +++ b/web/src/store/repos.ts @@ -32,6 +32,7 @@ export default defineStore({ async loadRepo(owner: string, name: string) { const repo = await apiClient.getRepo(owner, name); this.repos[repoSlug(repo)] = repo; + return repo; }, async loadRepos() { const repos = await apiClient.getRepoList(); diff --git a/web/src/views/repo/RepoWrapper.vue b/web/src/views/repo/RepoWrapper.vue index fd46838a7..a1becdaed 100644 --- a/web/src/views/repo/RepoWrapper.vue +++ b/web/src/views/repo/RepoWrapper.vue @@ -100,7 +100,14 @@ async function loadRepo() { return; } - await repoStore.loadRepo(repoOwner.value, repoName.value); + const apiRepo = await repoStore.loadRepo(repoOwner.value, repoName.value); + if (apiRepo.full_name !== `${repoOwner.value}/${repoName.value}`) { + await router.replace({ + name: route.name ? route.name : 'repo', + params: { repoOwner: apiRepo.owner, repoName: apiRepo.name }, + }); + return; + } await buildStore.loadBuilds(repoOwner.value, repoName.value); }