diff --git a/.gitignore b/.gitignore index fa6cbb454b..773b4726c0 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,4 @@ prime/ *.snap *.snap-build *_source.tar.bz2 +.DS_Store \ No newline at end of file diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 9bfddc97e8..dd14089d2b 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -808,3 +808,12 @@ IS_INPUT_FILE = false ENABLED = false ; If you want to add authorization, specify a token here TOKEN = + +[task] +; Task queue type, could be `channel` or `redis`. +QUEUE_TYPE = channel +; Task queue length, available only when `QUEUE_TYPE` is `channel`. +QUEUE_LENGTH = 1000 +; Task queue connction string, available only when `QUEUE_TYPE` is `redis`. +; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`. +QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0" \ No newline at end of file diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 198cff6f04..ed34be032b 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -514,9 +514,16 @@ Two special environment variables are passed to the render command: - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths. ## Time (`time`) + - `FORMAT`: Time format to diplay on UI. i.e. RFC1123 or 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: Default location of time on the UI, so that we can display correct user's time on UI. i.e. Shanghai/Asia +## Task (`task`) + +- `QUEUE_TYPE`: **channel**: Task queue type, could be `channel` or `redis`. +- `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`. +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`. + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index 541d66f4e9..01ba821a47 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -241,9 +241,16 @@ IS_INPUT_FILE = false - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。 ## Time (`time`) + - `FORMAT`: 显示在界面上的时间格式。比如： RFC1123 或者 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: 默认显示在界面上的时区，默认为本地时区。比如： Asia/Shanghai +## Task (`task`) + +- `QUEUE_TYPE`: **channel**: 任务队列类型，可以为 `channel` 或 `redis`。 +- `QUEUE_LENGTH`: **1000**: 任务队列长度，当 `QUEUE_TYPE` 为 `channel` 时有效。 +- `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串，当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码，则可以 `addrs=127.0.0.1:6379 password=123 db=0`。 + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 diff --git a/models/fixtures/repository.yml b/models/fixtures/repository.yml index 2e38c5e1dd..cf7d24c6cd 100644 --- a/models/fixtures/repository.yml +++ b/models/fixtures/repository.yml @@ -11,6 +11,7 @@ num_milestones: 3 num_closed_milestones: 1 num_watches: 3 + status: 0 - id: 2 @@ -24,6 +25,7 @@ num_closed_pulls: 0 num_stars: 1 close_issues_via_commit_in_any_branch: true + status: 0 - id: 3 @@ -36,6 +38,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 4 @@ -48,6 +51,7 @@ num_pulls: 0 num_closed_pulls: 0 num_stars: 1 + status: 0 - id: 5 @@ -61,6 +65,7 @@ num_closed_pulls: 0 num_watches: 0 is_mirror: true + status: 0 - id: 6 @@ -73,6 +78,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 7 @@ -85,6 +91,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 8 @@ -97,6 +104,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 9 @@ -109,6 +117,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 10 @@ -122,6 +131,7 @@ num_closed_pulls: 0 is_mirror: false num_forks: 1 + status: 0 - id: 11 @@ -135,6 +145,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 12 @@ -147,6 +158,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 13 @@ -159,6 +171,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 14 @@ -172,6 +185,7 @@ num_pulls: 0 num_closed_pulls: 0 is_mirror: false + status: 0 - id: 15 @@ -179,6 +193,7 @@ lower_name: repo15 name: repo15 is_empty: true + status: 0 - id: 16 @@ -191,6 +206,7 @@ num_pulls: 0 num_closed_pulls: 0 num_watches: 0 + status: 0 - id: 17 @@ -205,6 +221,7 @@ num_watches: 0 is_mirror: false is_fork: false + status: 0 - id: 18 @@ -218,6 +235,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 19 @@ -231,6 +249,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 20 @@ -244,6 +263,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 21 @@ -257,6 +277,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 22 @@ -270,6 +291,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 23 @@ -283,6 +305,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 24 @@ -296,6 +319,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: false + status: 0 - id: 25 @@ -310,6 +334,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 26 @@ -324,6 +349,7 @@ num_watches: 0 is_mirror: true is_fork: false + status: 0 - id: 27 @@ -339,6 +365,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 28 @@ -354,6 +381,7 @@ is_mirror: true num_forks: 1 is_fork: false + status: 0 - id: 29 @@ -368,6 +396,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 30 @@ -382,6 +411,7 @@ num_closed_pulls: 0 is_mirror: false is_fork: true + status: 0 - id: 31 @@ -392,6 +422,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 32 # org public repo @@ -403,6 +434,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 33 @@ -410,6 +442,7 @@ lower_name: utf8 name: utf8 is_private: false + status: 0 - id: 34 @@ -421,6 +454,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 35 @@ -432,6 +466,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 36 @@ -443,6 +478,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 37 @@ -454,6 +490,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 38 @@ -465,6 +502,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 39 @@ -476,6 +514,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 40 @@ -487,6 +526,7 @@ num_forks: 0 num_issues: 0 is_mirror: false + status: 0 - id: 41 @@ -519,4 +559,5 @@ num_stars: 0 num_forks: 0 num_issues: 0 - is_mirror: false \ No newline at end of file + is_mirror: false + status: 0 diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index e14437a04b..ef5cd377a6 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -252,6 +252,8 @@ var migrations = []Migration{ NewMigration("add repo_admin_change_team_access to user", addRepoAdminChangeTeamAccessColumnForUser), // v98 -> v99 NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases), + // v99 -> v100 + NewMigration("add task table and status column for repository table", addTaskTable), } // Migrate database to current version diff --git a/models/migrations/v99.go b/models/migrations/v99.go new file mode 100644 index 0000000000..3eb287af6c --- /dev/null +++ b/models/migrations/v99.go @@ -0,0 +1,34 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "github.com/go-xorm/xorm" +) + +func addTaskTable(x *xorm.Engine) error { + type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + RepoID int64 `xorm:"index"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` + } + + type Repository struct { + Status int `xorm:"NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(Task), new(Repository)) +} diff --git a/models/models.go b/models/models.go index e802a35a77..ea550cb839 100644 --- a/models/models.go +++ b/models/models.go @@ -112,6 +112,7 @@ func init() { new(OAuth2Application), new(OAuth2AuthorizationCode), new(OAuth2Grant), + new(Task), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 8db527477b..23b1c2ef52 100644 --- a/models/repo.go +++ b/models/repo.go @@ -126,6 +126,15 @@ func NewRepoContext() { RemoveAllWithNotice("Clean up repository temporary data", filepath.Join(setting.AppDataPath, "tmp")) } +// RepositoryStatus defines the status of repository +type RepositoryStatus int + +// all kinds of RepositoryStatus +const ( + RepositoryReady RepositoryStatus = iota // a normal repository + RepositoryBeingMigrated // repository is migrating +) + // Repository represents a git repository. type Repository struct { ID int64 `xorm:"pk autoincr"` @@ -156,9 +165,9 @@ type Repository struct { IsPrivate bool `xorm:"INDEX"` IsEmpty bool `xorm:"INDEX"` IsArchived bool `xorm:"INDEX"` - - IsMirror bool `xorm:"INDEX"` - *Mirror `xorm:"-"` + IsMirror bool `xorm:"INDEX"` + *Mirror `xorm:"-"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` ExternalMetas map[string]string `xorm:"-"` Units []*RepoUnit `xorm:"-"` @@ -197,6 +206,16 @@ func (repo *Repository) ColorFormat(s fmt.State) { repo.Name) } +// IsBeingMigrated indicates that repository is being migtated +func (repo *Repository) IsBeingMigrated() bool { + return repo.Status == RepositoryBeingMigrated +} + +// IsBeingCreated indicates that repository is being migrated or forked +func (repo *Repository) IsBeingCreated() bool { + return repo.IsBeingMigrated() +} + // AfterLoad is invoked from XORM after setting the values of all fields of this object. func (repo *Repository) AfterLoad() { // FIXME: use models migration to solve all at once. @@ -884,18 +903,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) { return repo.cloneLink(x, false) } -// MigrateRepoOptions contains the repository migrate options -type MigrateRepoOptions struct { - Name string - Description string - OriginalURL string - IsPrivate bool - IsMirror bool - RemoteAddr string - Wiki bool // include wiki repository - SyncReleasesWithTags bool // sync releases from tags -} - /* GitHub, GitLab, Gogs: *.wiki.git BitBucket: *.git/wiki @@ -915,20 +922,28 @@ func wikiRemoteURL(remote string) string { return "" } -// MigrateRepository migrates an existing repository from other project hosting. -func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) { - repo, err := CreateRepository(doer, u, CreateRepoOptions{ - Name: opts.Name, - Description: opts.Description, - OriginalURL: opts.OriginalURL, - IsPrivate: opts.IsPrivate, - IsMirror: opts.IsMirror, - }) - if err != nil { - return nil, err +// CheckCreateRepository check if could created a repository +func CheckCreateRepository(doer, u *User, name string) error { + if !doer.CanCreateRepo() { + return ErrReachLimitOfRepo{u.MaxRepoCreation} } - repoPath := RepoPath(u.Name, opts.Name) + if err := IsUsableRepoName(name); err != nil { + return err + } + + has, err := isRepositoryExist(x, u, name) + if err != nil { + return fmt.Errorf("IsRepositoryExist: %v", err) + } else if has { + return ErrRepoAlreadyExist{u.Name, name} + } + return nil +} + +// MigrateRepositoryGitData starts migrating git related data after created migrating repository +func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) { + repoPath := RepoPath(u.Name, opts.RepoName) if u.IsOrganization() { t, err := u.GetOwnerTeam() @@ -942,11 +957,12 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second - if err := os.RemoveAll(repoPath); err != nil { + var err error + if err = os.RemoveAll(repoPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{ + if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, @@ -955,8 +971,8 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err } if opts.Wiki { - wikiPath := WikiPath(u.Name, opts.Name) - wikiRemotePath := wikiRemoteURL(opts.RemoteAddr) + wikiPath := WikiPath(u.Name, opts.RepoName) + wikiRemotePath := wikiRemoteURL(opts.CloneAddr) if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) @@ -986,7 +1002,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err return repo, fmt.Errorf("git.IsEmpty: %v", err) } - if opts.SyncReleasesWithTags && !repo.IsEmpty { + if !opts.Releases && !repo.IsEmpty { // Try to get HEAD branch and set it as default branch. headBranch, err := gitRepo.GetHEADBranch() if err != nil { @@ -1005,7 +1021,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err log.Error("Failed to update size for repository: %v", err) } - if opts.IsMirror { + if opts.Mirror { if _, err = x.InsertOne(&Mirror{ RepoID: repo.ID, Interval: setting.Mirror.DefaultInterval, @@ -1143,6 +1159,7 @@ type CreateRepoOptions struct { IsPrivate bool IsMirror bool AutoInit bool + Status RepositoryStatus } func getRepoInitFile(tp, name string) ([]byte, error) { @@ -1410,6 +1427,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err IsPrivate: opts.IsPrivate, IsFsckEnabled: !opts.IsMirror, CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch, + Status: opts.Status, } sess := x.NewSession() @@ -1856,6 +1874,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { &CommitStatus{RepoID: repoID}, &RepoIndexerStatus{RepoID: repoID}, &Comment{RefRepoID: repoID}, + &Task{RepoID: repoID}, ); err != nil { return fmt.Errorf("deleteBeans: %v", err) } diff --git a/models/task.go b/models/task.go new file mode 100644 index 0000000000..cb878d387c --- /dev/null +++ b/models/task.go @@ -0,0 +1,240 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "encoding/json" + "fmt" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/builder" +) + +// Task represents a task +type Task struct { + ID int64 + DoerID int64 `xorm:"index"` // operator + Doer *User `xorm:"-"` + OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero + Owner *User `xorm:"-"` + RepoID int64 `xorm:"index"` + Repo *Repository `xorm:"-"` + Type structs.TaskType + Status structs.TaskStatus `xorm:"index"` + StartTime timeutil.TimeStamp + EndTime timeutil.TimeStamp + PayloadContent string `xorm:"TEXT"` + Errors string `xorm:"TEXT"` // if task failed, saved the error reason + Created timeutil.TimeStamp `xorm:"created"` +} + +// LoadRepo loads repository of the task +func (task *Task) LoadRepo() error { + return task.loadRepo(x) +} + +func (task *Task) loadRepo(e Engine) error { + if task.Repo != nil { + return nil + } + var repo Repository + has, err := e.ID(task.RepoID).Get(&repo) + if err != nil { + return err + } else if !has { + return ErrRepoNotExist{ + ID: task.RepoID, + } + } + task.Repo = &repo + return nil +} + +// LoadDoer loads do user +func (task *Task) LoadDoer() error { + if task.Doer != nil { + return nil + } + + var doer User + has, err := x.ID(task.DoerID).Get(&doer) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.DoerID, + } + } + task.Doer = &doer + + return nil +} + +// LoadOwner loads owner user +func (task *Task) LoadOwner() error { + if task.Owner != nil { + return nil + } + + var owner User + has, err := x.ID(task.OwnerID).Get(&owner) + if err != nil { + return err + } else if !has { + return ErrUserNotExist{ + UID: task.OwnerID, + } + } + task.Owner = &owner + + return nil +} + +// UpdateCols updates some columns +func (task *Task) UpdateCols(cols ...string) error { + _, err := x.ID(task.ID).Cols(cols...).Update(task) + return err +} + +// MigrateConfig returns task config when migrate repository +func (task *Task) MigrateConfig() (*structs.MigrateRepoOption, error) { + if task.Type == structs.TaskTypeMigrateRepo { + var opts structs.MigrateRepoOption + err := json.Unmarshal([]byte(task.PayloadContent), &opts) + if err != nil { + return nil, err + } + return &opts, nil + } + return nil, fmt.Errorf("Task type is %s, not Migrate Repo", task.Type.Name()) +} + +// ErrTaskDoesNotExist represents a "TaskDoesNotExist" kind of error. +type ErrTaskDoesNotExist struct { + ID int64 + RepoID int64 + Type structs.TaskType +} + +// IsErrTaskDoesNotExist checks if an error is a ErrTaskIsNotExist. +func IsErrTaskDoesNotExist(err error) bool { + _, ok := err.(ErrTaskDoesNotExist) + return ok +} + +func (err ErrTaskDoesNotExist) Error() string { + return fmt.Sprintf("task is not exist [id: %d, repo_id: %d, type: %d]", + err.ID, err.RepoID, err.Type) +} + +// GetMigratingTask returns the migrating task by repo's id +func GetMigratingTask(repoID int64) (*Task, error) { + var task = Task{ + RepoID: repoID, + Type: structs.TaskTypeMigrateRepo, + } + has, err := x.Get(&task) + if err != nil { + return nil, err + } else if !has { + return nil, ErrTaskDoesNotExist{0, repoID, task.Type} + } + return &task, nil +} + +// FindTaskOptions find all tasks +type FindTaskOptions struct { + Status int +} + +// ToConds generates conditions for database operation. +func (opts FindTaskOptions) ToConds() builder.Cond { + var cond = builder.NewCond() + if opts.Status >= 0 { + cond = cond.And(builder.Eq{"status": opts.Status}) + } + return cond +} + +// FindTasks find all tasks +func FindTasks(opts FindTaskOptions) ([]*Task, error) { + var tasks = make([]*Task, 0, 10) + err := x.Where(opts.ToConds()).Find(&tasks) + return tasks, err +} + +func createTask(e Engine, task *Task) error { + _, err := e.Insert(task) + return err +} + +// CreateMigrateTask creates a migrate task +func CreateMigrateTask(doer, u *User, opts base.MigrateOptions) (*Task, error) { + bs, err := json.Marshal(&opts) + if err != nil { + return nil, err + } + + var task = Task{ + DoerID: doer.ID, + OwnerID: u.ID, + Type: structs.TaskTypeMigrateRepo, + Status: structs.TaskStatusQueue, + PayloadContent: string(bs), + } + + if err := createTask(x, &task); err != nil { + return nil, err + } + + repo, err := CreateRepository(doer, u, CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + OriginalURL: opts.CloneAddr, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: RepositoryBeingMigrated, + }) + if err != nil { + task.EndTime = timeutil.TimeStampNow() + task.Status = structs.TaskStatusFailed + err2 := task.UpdateCols("end_time", "status") + if err2 != nil { + log.Error("UpdateCols Failed: %v", err2.Error()) + } + return nil, err + } + + task.RepoID = repo.ID + if err = task.UpdateCols("repo_id"); err != nil { + return nil, err + } + + return &task, nil +} + +// FinishMigrateTask updates database when migrate task finished +func FinishMigrateTask(task *Task) error { + task.Status = structs.TaskStatusFinished + task.EndTime = timeutil.TimeStampNow() + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil { + return err + } + task.Repo.Status = RepositoryReady + if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil { + return err + } + + return sess.Commit() +} diff --git a/modules/context/repo.go b/modules/context/repo.go index 3caf583f83..f4af19a0e8 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -146,6 +146,9 @@ func (r *Repository) FileExists(path string, branch string) (bool, error) { // GetEditorconfig returns the .editorconfig definition if found in the // HEAD of the default repo branch. func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) { + if r.GitRepo == nil { + return nil, nil + } commit, err := r.GitRepo.GetBranchCommit(r.Repository.DefaultBranch) if err != nil { return nil, err @@ -358,12 +361,6 @@ func RepoAssignment() macaron.Handler { return } - gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName)) - if err != nil { - ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err) - return - } - ctx.Repo.GitRepo = gitRepo ctx.Repo.RepoLink = repo.Link() ctx.Data["RepoLink"] = ctx.Repo.RepoLink ctx.Data["RepoRelPath"] = ctx.Repo.Owner.Name + "/" + ctx.Repo.Repository.Name @@ -373,13 +370,6 @@ func RepoAssignment() macaron.Handler { ctx.Data["RepoExternalIssuesLink"] = unit.ExternalTrackerConfig().ExternalTrackerURL } - tags, err := ctx.Repo.GitRepo.GetTags() - if err != nil { - ctx.ServerError("GetTags", err) - return - } - ctx.Data["Tags"] = tags - count, err := models.GetReleaseCountByRepoID(ctx.Repo.Repository.ID, models.FindReleasesOptions{ IncludeDrafts: false, IncludeTags: true, @@ -425,12 +415,25 @@ func RepoAssignment() macaron.Handler { } // repo is empty and display enable - if ctx.Repo.Repository.IsEmpty { + if ctx.Repo.Repository.IsEmpty || ctx.Repo.Repository.IsBeingCreated() { ctx.Data["BranchName"] = ctx.Repo.Repository.DefaultBranch return } - ctx.Data["TagName"] = ctx.Repo.TagName + gitRepo, err := git.OpenRepository(models.RepoPath(userName, repoName)) + if err != nil { + ctx.ServerError("RepoAssignment Invalid repo "+models.RepoPath(userName, repoName), err) + return + } + ctx.Repo.GitRepo = gitRepo + + tags, err := ctx.Repo.GitRepo.GetTags() + if err != nil { + ctx.ServerError("GetTags", err) + return + } + ctx.Data["Tags"] = tags + brs, err := ctx.Repo.GitRepo.GetBranches() if err != nil { ctx.ServerError("GetBranches", err) @@ -439,6 +442,8 @@ func RepoAssignment() macaron.Handler { ctx.Data["Branches"] = brs ctx.Data["BranchesCount"] = len(brs) + ctx.Data["TagName"] = ctx.Repo.TagName + // If not branch selected, try default one. // If default branch doesn't exists, fall back to some other branch. if len(ctx.Repo.BranchName) == 0 { diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index ba7fdc6815..2d180b61d9 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -5,22 +5,7 @@ package base -// MigrateOptions defines the way a repository gets migrated -type MigrateOptions struct { - RemoteURL string - AuthUsername string - AuthPassword string - Name string - Description string - OriginalURL string +import "code.gitea.io/gitea/modules/structs" - Wiki bool - Issues bool - Milestones bool - Labels bool - Releases bool - Comments bool - PullRequests bool - Private bool - Mirror bool -} +// MigrateOptions defines the way a repository gets migrated +type MigrateOptions = structs.MigrateRepoOption diff --git a/modules/migrations/gitea.go b/modules/migrations/gitea.go index 1edac47a6e..ab3b0b9f69 100644 --- a/modules/migrations/gitea.go +++ b/modules/migrations/gitea.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" gouuid "github.com/satori/go.uuid" @@ -90,16 +91,33 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate remoteAddr = u.String() } - r, err := models.MigrateRepository(g.doer, owner, models.MigrateRepoOptions{ - Name: g.repoName, - Description: repo.Description, - OriginalURL: repo.OriginalURL, - IsMirror: repo.IsMirror, - RemoteAddr: remoteAddr, - IsPrivate: repo.IsPrivate, - Wiki: opts.Wiki, - SyncReleasesWithTags: !opts.Releases, // if didn't get releases, then sync them from tags + var r *models.Repository + if opts.MigrateToRepoID <= 0 { + r, err = models.CreateRepository(g.doer, owner, models.CreateRepoOptions{ + Name: g.repoName, + Description: repo.Description, + OriginalURL: repo.OriginalURL, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: models.RepositoryBeingMigrated, + }) + } else { + r, err = models.GetRepositoryByID(opts.MigrateToRepoID) + } + if err != nil { + return err + } + + r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{ + RepoName: g.repoName, + Description: repo.Description, + Mirror: repo.IsMirror, + CloneAddr: remoteAddr, + Private: repo.IsPrivate, + Wiki: opts.Wiki, + Releases: opts.Releases, // if didn't get releases, then sync them from tags }) + g.repo = r if err != nil { return err diff --git a/modules/migrations/gitea_test.go b/modules/migrations/gitea_test.go index 88a3a6d218..73c119a15d 100644 --- a/modules/migrations/gitea_test.go +++ b/modules/migrations/gitea_test.go @@ -10,6 +10,7 @@ import ( "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" @@ -29,9 +30,9 @@ func TestGiteaUploadRepo(t *testing.T) { uploader = NewGiteaLocalUploader(user, user.Name, repoName) ) - err := migrateRepository(downloader, uploader, MigrateOptions{ - RemoteURL: "https://github.com/go-xorm/builder", - Name: repoName, + err := migrateRepository(downloader, uploader, structs.MigrateRepoOption{ + CloneAddr: "https://github.com/go-xorm/builder", + RepoName: repoName, AuthUsername: "", Wiki: true, diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 754f98941c..1c5d96c03d 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -34,7 +34,7 @@ type GithubDownloaderV3Factory struct { // Match returns ture if the migration remote URL matched this downloader factory func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) { - u, err := url.Parse(opts.RemoteURL) + u, err := url.Parse(opts.CloneAddr) if err != nil { return false, err } @@ -44,7 +44,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error // New returns a Downloader related to this factory according MigrateOptions func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) { - u, err := url.Parse(opts.RemoteURL) + u, err := url.Parse(opts.CloneAddr) if err != nil { return nil, err } diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 27782cb940..3f5c0d1118 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -6,6 +6,8 @@ package migrations import ( + "fmt" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" @@ -27,7 +29,7 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { var ( downloader base.Downloader - uploader = NewGiteaLocalUploader(doer, ownerName, opts.Name) + uploader = NewGiteaLocalUploader(doer, ownerName, opts.RepoName) ) for _, factory := range factories { @@ -50,14 +52,18 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt opts.Comments = false opts.Issues = false opts.PullRequests = false - downloader = NewPlainGitDownloader(ownerName, opts.Name, opts.RemoteURL) - log.Trace("Will migrate from git: %s", opts.RemoteURL) + downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr) + log.Trace("Will migrate from git: %s", opts.CloneAddr) } if err := migrateRepository(downloader, uploader, opts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) } + + if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.CloneAddr, err)); err2 != nil { + log.Error("create respotiry notice failed: ", err2) + } return nil, err } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 5e476854b2..8c61bdbb77 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1043,4 +1043,5 @@ func NewServices() { newNotifyMailService() newWebhookService() newIndexerService() + newTaskService() } diff --git a/modules/setting/task.go b/modules/setting/task.go new file mode 100644 index 0000000000..97704d4a4d --- /dev/null +++ b/modules/setting/task.go @@ -0,0 +1,25 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +var ( + // Task settings + Task = struct { + QueueType string + QueueLength int + QueueConnStr string + }{ + QueueType: ChannelQueueType, + QueueLength: 1000, + QueueConnStr: "addrs=127.0.0.1:6379 db=0", + } +) + +func newTaskService() { + sec := Cfg.Section("task") + Task.QueueType = sec.Key("QUEUE_TYPE").MustString(ChannelQueueType) + Task.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000) + Task.QueueConnStr = sec.Key("QUEUE_CONN_STR").MustString("addrs=127.0.0.1:6379 db=0") +} diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 87396d6ce9..57f1768a0b 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -162,8 +162,16 @@ type MigrateRepoOption struct { // required: true UID int `json:"uid" binding:"Required"` // required: true - RepoName string `json:"repo_name" binding:"Required"` - Mirror bool `json:"mirror"` - Private bool `json:"private"` - Description string `json:"description"` + RepoName string `json:"repo_name" binding:"Required"` + Mirror bool `json:"mirror"` + Private bool `json:"private"` + Description string `json:"description"` + Wiki bool + Issues bool + Milestones bool + Labels bool + Releases bool + Comments bool + PullRequests bool + MigrateToRepoID int64 } diff --git a/modules/structs/task.go b/modules/structs/task.go new file mode 100644 index 0000000000..e83d0437ce --- /dev/null +++ b/modules/structs/task.go @@ -0,0 +1,34 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package structs + +// TaskType defines task type +type TaskType int + +// all kinds of task types +const ( + TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk +) + +// Name returns the task type name +func (taskType TaskType) Name() string { + switch taskType { + case TaskTypeMigrateRepo: + return "Migrate Repository" + } + return "" +} + +// TaskStatus defines task status +type TaskStatus int + +// enumerate all the kinds of task status +const ( + TaskStatusQueue TaskStatus = iota // 0 task is queue + TaskStatusRunning // 1 task is running + TaskStatusStopped // 2 task is stopped + TaskStatusFailed // 3 task is failed + TaskStatusFinished // 4 task is finished +) diff --git a/modules/task/migrate.go b/modules/task/migrate.go new file mode 100644 index 0000000000..5d15a506d7 --- /dev/null +++ b/modules/task/migrate.go @@ -0,0 +1,120 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +func handleCreateError(owner *models.User, err error, name string) error { + switch { + case models.IsErrReachLimitOfRepo(err): + return fmt.Errorf("You have already reached your limit of %d repositories", owner.MaxCreationLimit()) + case models.IsErrRepoAlreadyExist(err): + return errors.New("The repository name is already used") + case models.IsErrNameReserved(err): + return fmt.Errorf("The repository name '%s' is reserved", err.(models.ErrNameReserved).Name) + case models.IsErrNamePatternNotAllowed(err): + return fmt.Errorf("The pattern '%s' is not allowed in a repository name", err.(models.ErrNamePatternNotAllowed).Pattern) + default: + return err + } +} + +func runMigrateTask(t *models.Task) (err error) { + defer func() { + if e := recover(); e != nil { + var buf bytes.Buffer + fmt.Fprintf(&buf, "Handler crashed with error: %v", log.Stack(2)) + + err = errors.New(buf.String()) + } + + if err == nil { + err = models.FinishMigrateTask(t) + if err == nil { + notification.NotifyMigrateRepository(t.Doer, t.Owner, t.Repo) + return + } + + log.Error("FinishMigrateTask failed: %s", err.Error()) + } + + t.EndTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusFailed + t.Errors = err.Error() + if err := t.UpdateCols("status", "errors", "end_time"); err != nil { + log.Error("Task UpdateCols failed: %s", err.Error()) + } + + if t.Repo != nil { + if errDelete := models.DeleteRepository(t.Doer, t.OwnerID, t.Repo.ID); errDelete != nil { + log.Error("DeleteRepository: %v", errDelete) + } + } + }() + + if err := t.LoadRepo(); err != nil { + return err + } + + // if repository is ready, then just finsih the task + if t.Repo.Status == models.RepositoryReady { + return nil + } + + if err := t.LoadDoer(); err != nil { + return err + } + if err := t.LoadOwner(); err != nil { + return err + } + t.StartTime = timeutil.TimeStampNow() + t.Status = structs.TaskStatusRunning + if err := t.UpdateCols("start_time", "status"); err != nil { + return err + } + + var opts *structs.MigrateRepoOption + opts, err = t.MigrateConfig() + if err != nil { + return err + } + + opts.MigrateToRepoID = t.RepoID + repo, err := migrations.MigrateRepository(t.Doer, t.Owner.Name, *opts) + if err == nil { + notification.NotifyMigrateRepository(t.Doer, t.Owner, repo) + + log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name) + return nil + } + + if models.IsErrRepoAlreadyExist(err) { + return errors.New("The repository name is already used") + } + + // remoteAddr may contain credentials, so we sanitize it + err = util.URLSanitizedError(err, opts.CloneAddr) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "could not read Username") { + return fmt.Errorf("Authentication failed: %v", err.Error()) + } else if strings.Contains(err.Error(), "fatal:") { + return fmt.Errorf("Migration failed: %v", err.Error()) + } + + return handleCreateError(t.Owner, err, "MigratePost") +} diff --git a/modules/task/queue.go b/modules/task/queue.go new file mode 100644 index 0000000000..ddee0b3d46 --- /dev/null +++ b/modules/task/queue.go @@ -0,0 +1,14 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import "code.gitea.io/gitea/models" + +// Queue defines an interface to run task queue +type Queue interface { + Run() error + Push(*models.Task) error + Stop() +} diff --git a/modules/task/queue_channel.go b/modules/task/queue_channel.go new file mode 100644 index 0000000000..da541f4755 --- /dev/null +++ b/modules/task/queue_channel.go @@ -0,0 +1,48 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +var ( + _ Queue = &ChannelQueue{} +) + +// ChannelQueue implements +type ChannelQueue struct { + queue chan *models.Task +} + +// NewChannelQueue create a memory channel queue +func NewChannelQueue(queueLen int) *ChannelQueue { + return &ChannelQueue{ + queue: make(chan *models.Task, queueLen), + } +} + +// Run starts to run the queue +func (c *ChannelQueue) Run() error { + for task := range c.queue { + err := Run(task) + if err != nil { + log.Error("Run task failed: %s", err.Error()) + } + } + return nil +} + +// Push will push the task ID to queue +func (c *ChannelQueue) Push(task *models.Task) error { + c.queue <- task + return nil +} + +// Stop stop the queue +func (c *ChannelQueue) Stop() { + close(c.queue) +} diff --git a/modules/task/queue_redis.go b/modules/task/queue_redis.go new file mode 100644 index 0000000000..127de0cdbf --- /dev/null +++ b/modules/task/queue_redis.go @@ -0,0 +1,130 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "encoding/json" + "errors" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + + "github.com/go-redis/redis" +) + +var ( + _ Queue = &RedisQueue{} +) + +type redisClient interface { + RPush(key string, args ...interface{}) *redis.IntCmd + LPop(key string) *redis.StringCmd + Ping() *redis.StatusCmd +} + +// RedisQueue redis queue +type RedisQueue struct { + client redisClient + queueName string + closeChan chan bool +} + +func parseConnStr(connStr string) (addrs, password string, dbIdx int, err error) { + fields := strings.Fields(connStr) + for _, f := range fields { + items := strings.SplitN(f, "=", 2) + if len(items) < 2 { + continue + } + switch strings.ToLower(items[0]) { + case "addrs": + addrs = items[1] + case "password": + password = items[1] + case "db": + dbIdx, err = strconv.Atoi(items[1]) + if err != nil { + return + } + } + } + return +} + +// NewRedisQueue creates single redis or cluster redis queue +func NewRedisQueue(addrs string, password string, dbIdx int) (*RedisQueue, error) { + dbs := strings.Split(addrs, ",") + var queue = RedisQueue{ + queueName: "task_queue", + closeChan: make(chan bool), + } + if len(dbs) == 0 { + return nil, errors.New("no redis host found") + } else if len(dbs) == 1 { + queue.client = redis.NewClient(&redis.Options{ + Addr: strings.TrimSpace(dbs[0]), // use default Addr + Password: password, // no password set + DB: dbIdx, // use default DB + }) + } else { + // cluster will ignore db + queue.client = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: dbs, + Password: password, + }) + } + if err := queue.client.Ping().Err(); err != nil { + return nil, err + } + return &queue, nil +} + +// Run starts to run the queue +func (r *RedisQueue) Run() error { + for { + select { + case <-r.closeChan: + return nil + case <-time.After(time.Millisecond * 100): + } + + bs, err := r.client.LPop(r.queueName).Bytes() + if err != nil { + if err != redis.Nil { + log.Error("LPop failed: %v", err) + } + time.Sleep(time.Millisecond * 100) + continue + } + + var task models.Task + err = json.Unmarshal(bs, &task) + if err != nil { + log.Error("Unmarshal task failed: %s", err.Error()) + } else { + err = Run(&task) + if err != nil { + log.Error("Run task failed: %s", err.Error()) + } + } + } +} + +// Push implements Queue +func (r *RedisQueue) Push(task *models.Task) error { + bs, err := json.Marshal(task) + if err != nil { + return err + } + return r.client.RPush(r.queueName, bs).Err() +} + +// Stop stop the queue +func (r *RedisQueue) Stop() { + r.closeChan <- true +} diff --git a/modules/task/task.go b/modules/task/task.go new file mode 100644 index 0000000000..64744afe7a --- /dev/null +++ b/modules/task/task.go @@ -0,0 +1,66 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package task + +import ( + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/structs" +) + +// taskQueue is a global queue of tasks +var taskQueue Queue + +// Run a task +func Run(t *models.Task) error { + switch t.Type { + case structs.TaskTypeMigrateRepo: + return runMigrateTask(t) + default: + return fmt.Errorf("Unknow task type: %d", t.Type) + } +} + +// Init will start the service to get all unfinished tasks and run them +func Init() error { + switch setting.Task.QueueType { + case setting.ChannelQueueType: + taskQueue = NewChannelQueue(setting.Task.QueueLength) + case setting.RedisQueueType: + var err error + addrs, pass, idx, err := parseConnStr(setting.Task.QueueConnStr) + if err != nil { + return err + } + taskQueue, err = NewRedisQueue(addrs, pass, idx) + if err != nil { + return err + } + default: + return fmt.Errorf("Unsupported task queue type: %v", setting.Task.QueueType) + } + + go func() { + if err := taskQueue.Run(); err != nil { + log.Error("taskQueue.Run end failed: %v", err) + } + }() + + return nil +} + +// MigrateRepository add migration repository to task +func MigrateRepository(doer, u *models.User, opts base.MigrateOptions) error { + task, err := models.CreateMigrateTask(doer, u, opts) + if err != nil { + return err + } + + return taskQueue.Push(task) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ca09b6120d..e6c5839a64 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -633,6 +633,8 @@ migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'g migrate.migrate_items_options = When migrating from github, input a username and migration options will be displayed. migrated_from = Migrated from %[2]s migrated_from_fake = Migrated From %[1]s +migrate.migrating = Migrating from %s ... +migrate.migrating_failed = Migrating from %s failed. mirror_from = mirror of forked_from = forked from diff --git a/public/img/loading.png b/public/img/loading.png new file mode 100644 index 0000000000..aac702cfd6 Binary files /dev/null and b/public/img/loading.png differ diff --git a/public/js/index.js b/public/js/index.js index 8a85ad9157..3b15ad8f18 100644 --- a/public/js/index.js +++ b/public/js/index.js @@ -241,6 +241,41 @@ function updateIssuesMeta(url, action, issueIds, elementId) { }) } +function initRepoStatusChecker() { + const migrating = $("#repo_migrating"); + $('#repo_migrating_failed').hide(); + if (migrating) { + const repo_name = migrating.attr('repo'); + if (typeof repo_name === 'undefined') { + return + } + $.ajax({ + type: "GET", + url: suburl +"/"+repo_name+"/status", + data: { + "_csrf": csrf, + }, + complete: function(xhr) { + if (xhr.status == 200) { + if (xhr.responseJSON) { + if (xhr.responseJSON["status"] == 0) { + location.reload(); + return + } + + setTimeout(function () { + initRepoStatusChecker() + }, 2000); + return + } + } + $('#repo_migrating_progress').hide(); + $('#repo_migrating_failed').show(); + } + }) + } +} + function initReactionSelector(parent) { let reactions = ''; if (!parent) { @@ -2219,6 +2254,7 @@ $(document).ready(function () { initIssueList(); initWipTitle(); initPullRequestReview(); + initRepoStatusChecker(); // Repo clone url. if ($('#repo-clone-url').length > 0) { diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index d8b06862a5..08c0635bc3 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -398,8 +398,8 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) { } var opts = migrations.MigrateOptions{ - RemoteURL: remoteAddr, - Name: form.RepoName, + CloneAddr: remoteAddr, + RepoName: form.RepoName, Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, diff --git a/routers/init.go b/routers/init.go index 1efddcfaa6..c37bbeb6b0 100644 --- a/routers/init.go +++ b/routers/init.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/markup/external" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" + "code.gitea.io/gitea/modules/task" "code.gitea.io/gitea/services/mailer" mirror_service "code.gitea.io/gitea/services/mirror" @@ -102,6 +103,9 @@ func GlobalInit() { mirror_service.InitSyncMirrors() models.InitDeliverHooks() models.InitTestPullRequests() + if err := task.Init(); err != nil { + log.Fatal("Failed to initialize task scheduler: %v", err) + } } if setting.EnableSQLite3 { log.Info("SQLite3 Supported") diff --git a/routers/private/serv.go b/routers/private/serv.go index 71c0f6ea2c..c4508b4cb5 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -119,6 +119,15 @@ func ServCommand(ctx *macaron.Context) { repo.OwnerName = ownerName results.RepoID = repo.ID + if repo.IsBeingCreated() { + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "results": results, + "type": "InternalServerError", + "err": "Repository is being created, you could retry after it finished", + }) + return + } + // We can shortcut at this point if the repo is a mirror if mode > models.AccessModeRead && repo.IsMirror { ctx.JSON(http.StatusUnauthorized, map[string]interface{}{ diff --git a/routers/repo/repo.go b/routers/repo/repo.go index b67384d721..bfd0c771b0 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/task" "code.gitea.io/gitea/modules/util" "github.com/unknwon/com" @@ -133,8 +134,6 @@ func Create(ctx *context.Context) { func handleCreateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form interface{}) { switch { - case migrations.IsRateLimitError(err): - ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form) case models.IsErrReachLimitOfRepo(err): ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form) case models.IsErrRepoAlreadyExist(err): @@ -221,6 +220,40 @@ func Migrate(ctx *context.Context) { ctx.HTML(200, tplMigrate) } +func handleMigrateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form *auth.MigrateRepoForm) { + switch { + case migrations.IsRateLimitError(err): + ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form) + case migrations.IsTwoFactorAuthError(err): + ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form) + case models.IsErrReachLimitOfRepo(err): + ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form) + case models.IsErrRepoAlreadyExist(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tpl, form) + case models.IsErrNameReserved(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tpl, form) + case models.IsErrNamePatternNotAllowed(err): + ctx.Data["Err_RepoName"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form) + default: + remoteAddr, _ := form.ParseRemoteAddr(owner) + err = util.URLSanitizedError(err, remoteAddr) + if strings.Contains(err.Error(), "Authentication failed") || + strings.Contains(err.Error(), "Bad credentials") || + strings.Contains(err.Error(), "could not read Username") { + ctx.Data["Err_Auth"] = true + ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tpl, form) + } else if strings.Contains(err.Error(), "fatal:") { + ctx.Data["Err_CloneAddr"] = true + ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tpl, form) + } else { + ctx.ServerError(name, err) + } + } +} + // MigratePost response for migrating from external git repository func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { ctx.Data["Title"] = ctx.Tr("new_migrate") @@ -258,8 +291,8 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { } var opts = migrations.MigrateOptions{ - RemoteURL: remoteAddr, - Name: form.RepoName, + CloneAddr: remoteAddr, + RepoName: form.RepoName, Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, @@ -282,47 +315,19 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) { opts.Releases = false } - repo, err := migrations.MigrateRepository(ctx.User, ctxUser.Name, opts) - if err == nil { - notification.NotifyCreateRepository(ctx.User, ctxUser, repo) - - log.Trace("Repository migrated [%d]: %s/%s successfully", repo.ID, ctxUser.Name, form.RepoName) - ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + form.RepoName) + err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName) + if err != nil { + handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form) return } - switch { - case models.IsErrReachLimitOfRepo(err): - ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", ctxUser.MaxCreationLimit()), tplMigrate, &form) - case models.IsErrNameReserved(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tplMigrate, &form) - case models.IsErrRepoAlreadyExist(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tplMigrate, &form) - case models.IsErrNamePatternNotAllowed(err): - ctx.Data["Err_RepoName"] = true - ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tplMigrate, &form) - case migrations.IsRateLimitError(err): - ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tplMigrate, &form) - case migrations.IsTwoFactorAuthError(err): - ctx.Data["Err_Auth"] = true - ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tplMigrate, &form) - default: - // remoteAddr may contain credentials, so we sanitize it - err = util.URLSanitizedError(err, remoteAddr) - if strings.Contains(err.Error(), "Authentication failed") || - strings.Contains(err.Error(), "Bad credentials") || - strings.Contains(err.Error(), "could not read Username") { - ctx.Data["Err_Auth"] = true - ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tplMigrate, &form) - } else if strings.Contains(err.Error(), "fatal:") { - ctx.Data["Err_CloneAddr"] = true - ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tplMigrate, &form) - } else { - ctx.ServerError("MigratePost", err) - } + err = task.MigrateRepository(ctx.User, ctxUser, opts) + if err == nil { + ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + opts.RepoName) + return } + + handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form) } // Action response for actions to a repository @@ -460,3 +465,19 @@ func Download(ctx *context.Context) { ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } + +// Status returns repository's status +func Status(ctx *context.Context) { + task, err := models.GetMigratingTask(ctx.Repo.Repository.ID) + if err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err, + }) + return + } + + ctx.JSON(200, map[string]interface{}{ + "status": ctx.Repo.Repository.Status, + "err": task.Errors, + }) +} diff --git a/routers/repo/view.go b/routers/repo/view.go index 1967b511ca..c4e6a69220 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -11,6 +11,7 @@ import ( "fmt" gotemplate "html/template" "io/ioutil" + "net/url" "path" "strings" @@ -31,6 +32,7 @@ const ( tplRepoHome base.TplName = "repo/home" tplWatchers base.TplName = "repo/watchers" tplForks base.TplName = "repo/forks" + tplMigrating base.TplName = "repo/migrating" ) func renderDirectory(ctx *context.Context, treeLink string) { @@ -356,9 +358,37 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } } +func safeURL(address string) string { + u, err := url.Parse(address) + if err != nil { + return address + } + u.User = nil + return u.String() +} + // Home render repository home page func Home(ctx *context.Context) { if len(ctx.Repo.Units) > 0 { + if ctx.Repo.Repository.IsBeingCreated() { + task, err := models.GetMigratingTask(ctx.Repo.Repository.ID) + if err != nil { + ctx.ServerError("models.GetMigratingTask", err) + return + } + cfg, err := task.MigrateConfig() + if err != nil { + ctx.ServerError("task.MigrateConfig", err) + return + } + + ctx.Data["Repo"] = ctx.Repo + ctx.Data["MigrateTask"] = task + ctx.Data["CloneAddr"] = safeURL(cfg.CloneAddr) + ctx.HTML(200, tplMigrating) + return + } + var firstUnit *models.Unit for _, repoUnit := range ctx.Repo.Units { if repoUnit.Type == models.UnitTypeCode { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 11f2029226..8dfcdb9c9b 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -845,6 +845,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/archive/*", repo.MustBeNotEmpty, reqRepoCodeReader, repo.Download) + m.Get("/status", reqRepoCodeReader, repo.Status) + m.Group("/branches", func() { m.Get("", repo.Branches) }, repo.MustBeNotEmpty, context.RepoRef(), reqRepoCodeReader) diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go index 76bd4c72f7..9ad11b7265 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_test.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/structs" release_service "code.gitea.io/gitea/services/release" "github.com/stretchr/testify/assert" @@ -26,16 +27,26 @@ func TestRelease_MirrorDelete(t *testing.T) { repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) repoPath := models.RepoPath(user.Name, repo.Name) - migrationOptions := models.MigrateRepoOptions{ - Name: "test_mirror", - Description: "Test mirror", - IsPrivate: false, - IsMirror: true, - RemoteAddr: repoPath, - Wiki: true, - SyncReleasesWithTags: true, + opts := structs.MigrateRepoOption{ + RepoName: "test_mirror", + Description: "Test mirror", + Private: false, + Mirror: true, + CloneAddr: repoPath, + Wiki: true, + Releases: false, } - mirror, err := models.MigrateRepository(user, user, migrationOptions) + + mirrorRepo, err := models.CreateRepository(user, user, models.CreateRepoOptions{ + Name: opts.RepoName, + Description: opts.Description, + IsPrivate: opts.Private, + IsMirror: opts.Mirror, + Status: models.RepositoryBeingMigrated, + }) + assert.NoError(t, err) + + mirror, err := models.MigrateRepositoryGitData(user, user, mirrorRepo, opts) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath) diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index fc7f1b660c..9fb3e32899 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -16,93 +16,95 @@ {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{MirrorAddress $.Mirror}}
{{end}} {{if .IsFork}}
{{$.i18n.Tr "repo.forked_from"}} {{SubStr .BaseRepo.RelLink 1 -1}}
{{end}} -
-
- - {{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}} - - - {{.NumWatches}} - -
-
- - {{if $.IsStaringRepo}}{{$.i18n.Tr "repo.unstar"}}{{else}}{{$.i18n.Tr "repo.star"}}{{end}} - - - {{.NumStars}} - -
- {{if and (not .IsEmpty) ($.Permission.CanRead $.UnitTypeCode)}} -
- - {{$.i18n.Tr "repo.fork"}} + {{if not .IsBeingCreated}} +
+
+ + {{if $.IsWatchingRepo}}{{$.i18n.Tr "repo.unwatch"}}{{else}}{{$.i18n.Tr "repo.watch"}}{{end}} - - {{.NumForks}} + + {{.NumWatches}} + +
+
+ + {{if $.IsStaringRepo}}{{$.i18n.Tr "repo.unstar"}}{{else}}{{$.i18n.Tr "repo.star"}}{{end}} + + + {{.NumStars}} + +
+ {{if and (not .IsEmpty) ($.Permission.CanRead $.UnitTypeCode)}} +
+ + {{$.i18n.Tr "repo.fork"}} + + + {{.NumForks}} + +
+ {{end}} +
+ {{end}} +
+
+{{end}} +
+ {{if not .Repository.IsBeingCreated}} +
+ {{if .Permission.CanRead $.UnitTypeCode}} + + {{.i18n.Tr "repo.code"}} + + {{end}} + + {{if .Permission.CanRead $.UnitTypeIssues}} + + {{.i18n.Tr "repo.issues"}} {{.Repository.NumOpenIssues}} + + {{end}} + + {{if .Permission.CanRead $.UnitTypeExternalTracker}} + + {{.i18n.Tr "repo.issues"}} + + {{end}} + + {{if and .Repository.CanEnablePulls (.Permission.CanRead $.UnitTypePullRequests)}} + + {{.i18n.Tr "repo.pulls"}} {{.Repository.NumOpenPulls}} + + {{end}} + + {{if and (.Permission.CanRead $.UnitTypeReleases) (not .IsEmptyRepo) }} + + {{.i18n.Tr "repo.releases"}} {{.Repository.NumReleases}} + + {{end}} + + {{if or (.Permission.CanRead $.UnitTypeWiki) (.Permission.CanRead $.UnitTypeExternalWiki)}} + + {{.i18n.Tr "repo.wiki"}} + + {{end}} + + {{if and (.Permission.CanReadAny $.UnitTypePullRequests $.UnitTypeIssues $.UnitTypeReleases) (not .IsEmptyRepo)}} + + {{.i18n.Tr "repo.activity"}} + + {{end}} + + {{template "custom/extra_tabs" .}} + + {{if .Permission.IsAdmin}} +
+ + {{.i18n.Tr "repo.settings"}}
{{end}}
-
- -{{end}} - -
-
- {{if .Permission.CanRead $.UnitTypeCode}} - - {{.i18n.Tr "repo.code"}} - - {{end}} - - {{if .Permission.CanRead $.UnitTypeIssues}} - - {{.i18n.Tr "repo.issues"}} {{.Repository.NumOpenIssues}} - - {{end}} - - {{if .Permission.CanRead $.UnitTypeExternalTracker}} - - {{.i18n.Tr "repo.issues"}} - - {{end}} - - {{if and .Repository.CanEnablePulls (.Permission.CanRead $.UnitTypePullRequests)}} - - {{.i18n.Tr "repo.pulls"}} {{.Repository.NumOpenPulls}} - - {{end}} - - {{if and (.Permission.CanRead $.UnitTypeReleases) (not .IsEmptyRepo) }} - - {{.i18n.Tr "repo.releases"}} {{.Repository.NumReleases}} - - {{end}} - - {{if or (.Permission.CanRead $.UnitTypeWiki) (.Permission.CanRead $.UnitTypeExternalWiki)}} - - {{.i18n.Tr "repo.wiki"}} - - {{end}} - - {{if and (.Permission.CanReadAny $.UnitTypePullRequests $.UnitTypeIssues $.UnitTypeReleases) (not .IsEmptyRepo)}} - - {{.i18n.Tr "repo.activity"}} - - {{end}} - - {{template "custom/extra_tabs" .}} - - {{if .Permission.IsAdmin}} -
- - {{.i18n.Tr "repo.settings"}} - -
- {{end}} -
+ {{end}}
- diff --git a/templates/repo/migrating.tmpl b/templates/repo/migrating.tmpl new file mode 100644 index 0000000000..34031d5653 --- /dev/null +++ b/templates/repo/migrating.tmpl @@ -0,0 +1,31 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+
+
+ {{template "base/alert" .}} +
+
+
+
+ +
+
+
+
+
+
+

{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}

+
+
+

{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}

+
+
+
+
+
+
+
+
+{{template "base/footer" .}}