From a21128a734636c2a18431cfc1742e8a7cf165f58 Mon Sep 17 00:00:00 2001 From: Gusted Date: Mon, 12 Aug 2024 17:16:55 +0200 Subject: [PATCH] [CHORE] Drop `go-git` support See https://codeberg.org/forgejo/discussions/issues/164 for the rationale and discussion of this change. Everything related to the `go-git` dependency is dropped (Only a single instance is left in a test file to test for an XSS, it requires crafting an commit that Git itself refuses to craft). `_gogit` files have been removed entirely, `go:build: !gogit` is removed, `XXX_nogogit.go` files either have been renamed or had their code being merged into the `XXX.go` file. --- Makefile | 3 - go.mod | 4 +- modules/git/blob.go | 121 ++++++- modules/git/blob_gogit.go | 32 -- modules/git/blob_nogogit.go | 118 ------- modules/git/commit_convert_gogit.go | 75 ----- modules/git/commit_info.go | 164 ++++++++++ modules/git/commit_info_gogit.go | 304 ------------------ modules/git/commit_info_nogogit.go | 170 ---------- modules/git/commit_sha256_test.go | 2 - modules/git/git.go | 4 +- modules/git/last_commit_cache.go | 45 +++ modules/git/last_commit_cache_gogit.go | 65 ---- modules/git/last_commit_cache_nogogit.go | 54 ---- modules/git/notes.go | 85 +++++ modules/git/notes_gogit.go | 89 ----- modules/git/notes_nogogit.go | 91 ------ modules/git/object_id_gogit.go | 30 -- modules/git/{parse_nogogit.go => parse.go} | 2 - modules/git/parse_gogit.go | 96 ------ modules/git/parse_gogit_test.go | 79 ----- .../{parse_nogogit_test.go => parse_test.go} | 2 - .../git/pipeline/{lfs_nogogit.go => lfs.go} | 25 +- modules/git/pipeline/lfs_common.go | 32 -- modules/git/pipeline/lfs_gogit.go | 146 --------- modules/git/repo_base.go | 114 ++++++- modules/git/repo_base_gogit.go | 107 ------ modules/git/repo_base_nogogit.go | 122 ------- modules/git/repo_blob.go | 13 - modules/git/repo_blob_gogit.go | 22 -- modules/git/repo_blob_nogogit.go | 16 - modules/git/repo_branch.go | 182 +++++++++++ modules/git/repo_branch_gogit.go | 147 --------- modules/git/repo_branch_nogogit.go | 194 ----------- modules/git/repo_commit.go | 149 +++++++++ modules/git/repo_commit_gogit.go | 111 ------- modules/git/repo_commit_nogogit.go | 161 ---------- modules/git/repo_commitgraph_gogit.go | 37 --- modules/git/repo_language_stats.go | 200 ++++++++++++ modules/git/repo_language_stats_gogit.go | 194 ----------- modules/git/repo_language_stats_nogogit.go | 210 ------------ modules/git/repo_language_stats_test.go | 2 - modules/git/repo_ref.go | 77 +++++ modules/git/repo_ref_gogit.go | 51 --- modules/git/repo_ref_nogogit.go | 87 ----- modules/git/repo_tag.go | 122 +++++++ modules/git/repo_tag_gogit.go | 135 -------- modules/git/repo_tag_nogogit.go | 134 -------- modules/git/repo_tree.go | 86 +++++ modules/git/repo_tree_gogit.go | 53 --- modules/git/repo_tree_nogogit.go | 95 ------ modules/git/signature.go | 18 ++ modules/git/signature_gogit.go | 14 - modules/git/signature_nogogit.go | 30 -- modules/git/tree.go | 111 +++++++ modules/git/tree_blob.go | 43 ++- modules/git/tree_blob_gogit.go | 65 ---- modules/git/tree_blob_nogogit.go | 49 --- modules/git/tree_entry.go | 90 ++++++ modules/git/tree_entry_gogit.go | 95 ------ modules/git/tree_entry_nogogit.go | 96 ------ modules/git/tree_entry_test.go | 103 ------ modules/git/tree_gogit.go | 98 ------ modules/git/tree_nogogit.go | 121 ------- modules/git/utils_test.go | 2 +- modules/gitrepo/{walk_nogogit.go => walk.go} | 2 - modules/gitrepo/walk_gogit.go | 40 --- ..._scanner_nogogit.go => pointer_scanner.go} | 2 - modules/lfs/pointer_scanner_gogit.go | 62 ---- release-notes/4941.md | 1 + 70 files changed, 1632 insertions(+), 4069 deletions(-) delete mode 100644 modules/git/blob_gogit.go delete mode 100644 modules/git/blob_nogogit.go delete mode 100644 modules/git/commit_convert_gogit.go delete mode 100644 modules/git/commit_info_gogit.go delete mode 100644 modules/git/commit_info_nogogit.go delete mode 100644 modules/git/last_commit_cache_gogit.go delete mode 100644 modules/git/last_commit_cache_nogogit.go delete mode 100644 modules/git/notes_gogit.go delete mode 100644 modules/git/notes_nogogit.go delete mode 100644 modules/git/object_id_gogit.go rename modules/git/{parse_nogogit.go => parse.go} (99%) delete mode 100644 modules/git/parse_gogit.go delete mode 100644 modules/git/parse_gogit_test.go rename modules/git/{parse_nogogit_test.go => parse_test.go} (99%) rename modules/git/pipeline/{lfs_nogogit.go => lfs.go} (90%) delete mode 100644 modules/git/pipeline/lfs_common.go delete mode 100644 modules/git/pipeline/lfs_gogit.go delete mode 100644 modules/git/repo_base_gogit.go delete mode 100644 modules/git/repo_base_nogogit.go delete mode 100644 modules/git/repo_blob.go delete mode 100644 modules/git/repo_blob_gogit.go delete mode 100644 modules/git/repo_blob_nogogit.go delete mode 100644 modules/git/repo_branch_gogit.go delete mode 100644 modules/git/repo_branch_nogogit.go delete mode 100644 modules/git/repo_commit_gogit.go delete mode 100644 modules/git/repo_commit_nogogit.go delete mode 100644 modules/git/repo_commitgraph_gogit.go delete mode 100644 modules/git/repo_language_stats_gogit.go delete mode 100644 modules/git/repo_language_stats_nogogit.go delete mode 100644 modules/git/repo_ref_gogit.go delete mode 100644 modules/git/repo_ref_nogogit.go delete mode 100644 modules/git/repo_tag_gogit.go delete mode 100644 modules/git/repo_tag_nogogit.go delete mode 100644 modules/git/repo_tree_gogit.go delete mode 100644 modules/git/repo_tree_nogogit.go delete mode 100644 modules/git/signature_gogit.go delete mode 100644 modules/git/signature_nogogit.go delete mode 100644 modules/git/tree_blob_gogit.go delete mode 100644 modules/git/tree_blob_nogogit.go delete mode 100644 modules/git/tree_entry_gogit.go delete mode 100644 modules/git/tree_entry_nogogit.go delete mode 100644 modules/git/tree_entry_test.go delete mode 100644 modules/git/tree_gogit.go delete mode 100644 modules/git/tree_nogogit.go rename modules/gitrepo/{walk_nogogit.go => walk.go} (95%) delete mode 100644 modules/gitrepo/walk_gogit.go rename modules/lfs/{pointer_scanner_nogogit.go => pointer_scanner.go} (99%) delete mode 100644 modules/lfs/pointer_scanner_gogit.go create mode 100644 release-notes/4941.md diff --git a/Makefile b/Makefile index 5190a80c18..754a973136 100644 --- a/Makefile +++ b/Makefile @@ -836,9 +836,6 @@ $(DIST_DIRS): .PHONY: release-windows release-windows: | $(DIST_DIRS) CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'osusergo $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets 'windows/*' -out gitea-$(VERSION) . -ifeq (,$(findstring gogit,$(TAGS))) - CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'osusergo gogit $(TAGS)' -ldflags '-linkmode external -extldflags "-static" $(LDFLAGS)' -targets 'windows/*' -out gitea-$(VERSION)-gogit . -endif .PHONY: release-linux release-linux: | $(DIST_DIRS) diff --git a/go.mod b/go.mod index 78f8e988d2..7d4574e60e 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,6 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/editorconfig/editorconfig-core-go/v2 v2.6.2 github.com/emersion/go-imap v1.2.1 - github.com/emirpasic/gods v1.18.1 github.com/felixge/fgprof v0.9.4 github.com/fsnotify/fsnotify v1.7.0 github.com/gliderlabs/ssh v0.3.7 @@ -42,7 +41,6 @@ require ( github.com/go-co-op/gocron v1.37.0 github.com/go-enry/go-enry/v2 v2.8.8 github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e - github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.11.0 github.com/go-ldap/ldap/v3 v3.4.6 github.com/go-sql-driver/mysql v1.8.1 @@ -172,6 +170,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.16.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect @@ -181,6 +180,7 @@ require ( github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-openapi/analysis v0.22.2 // indirect github.com/go-openapi/errors v0.21.0 // indirect diff --git a/modules/git/blob.go b/modules/git/blob.go index bcecb42e16..3de8ce8e90 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -5,15 +5,119 @@ package git import ( + "bufio" "bytes" "encoding/base64" "io" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" ) -// This file contains common functions between the gogit and !gogit variants for git Blobs +// Blob represents a Git object. +type Blob struct { + ID ObjectID + + gotSize bool + size int64 + name string + repo *Repository +} + +// DataAsync gets a ReadCloser for the contents of a blob without reading it all. +// Calling the Close function on the result will discard all unread output. +func (b *Blob) DataAsync() (io.ReadCloser, error) { + wr, rd, cancel := b.repo.CatFileBatch(b.repo.Ctx) + + _, err := wr.Write([]byte(b.ID.String() + "\n")) + if err != nil { + cancel() + return nil, err + } + _, _, size, err := ReadBatchLine(rd) + if err != nil { + cancel() + return nil, err + } + b.gotSize = true + b.size = size + + if size < 4096 { + bs, err := io.ReadAll(io.LimitReader(rd, size)) + defer cancel() + if err != nil { + return nil, err + } + _, err = rd.Discard(1) + return io.NopCloser(bytes.NewReader(bs)), err + } + + return &blobReader{ + rd: rd, + n: size, + cancel: cancel, + }, nil +} + +// Size returns the uncompressed size of the blob +func (b *Blob) Size() int64 { + if b.gotSize { + return b.size + } + + wr, rd, cancel := b.repo.CatFileBatchCheck(b.repo.Ctx) + defer cancel() + _, err := wr.Write([]byte(b.ID.String() + "\n")) + if err != nil { + log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) + return 0 + } + _, _, b.size, err = ReadBatchLine(rd) + if err != nil { + log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) + return 0 + } + + b.gotSize = true + + return b.size +} + +type blobReader struct { + rd *bufio.Reader + n int64 + cancel func() +} + +func (b *blobReader) Read(p []byte) (n int, err error) { + if b.n <= 0 { + return 0, io.EOF + } + if int64(len(p)) > b.n { + p = p[0:b.n] + } + n, err = b.rd.Read(p) + b.n -= int64(n) + return n, err +} + +// Close implements io.Closer +func (b *blobReader) Close() error { + if b.rd == nil { + return nil + } + + defer b.cancel() + + if err := DiscardFull(b.rd, b.n+1); err != nil { + return err + } + + b.rd = nil + + return nil +} // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { @@ -100,3 +204,18 @@ func (b *Blob) GuessContentType() (typesniffer.SniffedType, error) { return typesniffer.DetectContentTypeFromReader(r) } + +// GetBlob finds the blob object in the repository. +func (repo *Repository) GetBlob(idStr string) (*Blob, error) { + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + if id.IsZero() { + return nil, ErrNotExist{id.String(), ""} + } + return &Blob{ + ID: id, + repo: repo, + }, nil +} diff --git a/modules/git/blob_gogit.go b/modules/git/blob_gogit.go deleted file mode 100644 index 8c79c067c1..0000000000 --- a/modules/git/blob_gogit.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "io" - - "github.com/go-git/go-git/v5/plumbing" -) - -// Blob represents a Git object. -type Blob struct { - ID ObjectID - - gogitEncodedObj plumbing.EncodedObject - name string -} - -// DataAsync gets a ReadCloser for the contents of a blob without reading it all. -// Calling the Close function on the result will discard all unread output. -func (b *Blob) DataAsync() (io.ReadCloser, error) { - return b.gogitEncodedObj.Reader() -} - -// Size returns the uncompressed size of the blob -func (b *Blob) Size() int64 { - return b.gogitEncodedObj.Size() -} diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go deleted file mode 100644 index 945a6bc432..0000000000 --- a/modules/git/blob_nogogit.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "bufio" - "bytes" - "io" - - "code.gitea.io/gitea/modules/log" -) - -// Blob represents a Git object. -type Blob struct { - ID ObjectID - - gotSize bool - size int64 - name string - repo *Repository -} - -// DataAsync gets a ReadCloser for the contents of a blob without reading it all. -// Calling the Close function on the result will discard all unread output. -func (b *Blob) DataAsync() (io.ReadCloser, error) { - wr, rd, cancel := b.repo.CatFileBatch(b.repo.Ctx) - - _, err := wr.Write([]byte(b.ID.String() + "\n")) - if err != nil { - cancel() - return nil, err - } - _, _, size, err := ReadBatchLine(rd) - if err != nil { - cancel() - return nil, err - } - b.gotSize = true - b.size = size - - if size < 4096 { - bs, err := io.ReadAll(io.LimitReader(rd, size)) - defer cancel() - if err != nil { - return nil, err - } - _, err = rd.Discard(1) - return io.NopCloser(bytes.NewReader(bs)), err - } - - return &blobReader{ - rd: rd, - n: size, - cancel: cancel, - }, nil -} - -// Size returns the uncompressed size of the blob -func (b *Blob) Size() int64 { - if b.gotSize { - return b.size - } - - wr, rd, cancel := b.repo.CatFileBatchCheck(b.repo.Ctx) - defer cancel() - _, err := wr.Write([]byte(b.ID.String() + "\n")) - if err != nil { - log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) - return 0 - } - _, _, b.size, err = ReadBatchLine(rd) - if err != nil { - log.Debug("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repo.Path, err) - return 0 - } - - b.gotSize = true - - return b.size -} - -type blobReader struct { - rd *bufio.Reader - n int64 - cancel func() -} - -func (b *blobReader) Read(p []byte) (n int, err error) { - if b.n <= 0 { - return 0, io.EOF - } - if int64(len(p)) > b.n { - p = p[0:b.n] - } - n, err = b.rd.Read(p) - b.n -= int64(n) - return n, err -} - -// Close implements io.Closer -func (b *blobReader) Close() error { - if b.rd == nil { - return nil - } - - defer b.cancel() - - if err := DiscardFull(b.rd, b.n+1); err != nil { - return err - } - - b.rd = nil - - return nil -} diff --git a/modules/git/commit_convert_gogit.go b/modules/git/commit_convert_gogit.go deleted file mode 100644 index c413465656..0000000000 --- a/modules/git/commit_convert_gogit.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "fmt" - "strings" - - "github.com/go-git/go-git/v5/plumbing/object" -) - -func convertPGPSignature(c *object.Commit) *ObjectSignature { - if c.PGPSignature == "" { - return nil - } - - var w strings.Builder - var err error - - if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil { - return nil - } - - for _, parent := range c.ParentHashes { - if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil { - return nil - } - } - - if _, err = fmt.Fprint(&w, "author "); err != nil { - return nil - } - - if err = c.Author.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil { - return nil - } - - if err = c.Committer.Encode(&w); err != nil { - return nil - } - - if c.Encoding != "" && c.Encoding != "UTF-8" { - if _, err = fmt.Fprintf(&w, "\nencoding %s\n", c.Encoding); err != nil { - return nil - } - } - - if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil { - return nil - } - - return &ObjectSignature{ - Signature: c.PGPSignature, - Payload: w.String(), - } -} - -func convertCommit(c *object.Commit) *Commit { - return &Commit{ - ID: ParseGogitHash(c.Hash), - CommitMessage: c.Message, - Committer: &c.Committer, - Author: &c.Author, - Signature: convertPGPSignature(c), - Parents: ParseGogitHashArray(c.ParentHashes), - } -} diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index c740a4e13e..a26d749320 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -3,9 +3,173 @@ package git +import ( + "context" + "fmt" + "io" + "path" + "sort" + + "code.gitea.io/gitea/modules/log" +) + // CommitInfo describes the first commit with the provided entry type CommitInfo struct { Entry *TreeEntry Commit *Commit SubModuleFile *SubModuleFile } + +// GetCommitsInfo gets information of all commits that are corresponding to these entries +func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { + entryPaths := make([]string, len(tes)+1) + // Get the commit for the treePath itself + entryPaths[0] = "" + for i, entry := range tes { + entryPaths[i+1] = entry.Name() + } + + var err error + + var revs map[string]*Commit + if commit.repo.LastCommitCache != nil { + var unHitPaths []string + revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache) + if err != nil { + return nil, nil, err + } + if len(unHitPaths) > 0 { + sort.Strings(unHitPaths) + commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths) + if err != nil { + return nil, nil, err + } + + for pth, found := range commits { + revs[pth] = found + } + } + } else { + sort.Strings(entryPaths) + revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths) + } + if err != nil { + return nil, nil, err + } + + commitsInfo := make([]CommitInfo, len(tes)) + for i, entry := range tes { + commitsInfo[i] = CommitInfo{ + Entry: entry, + } + + // Check if we have found a commit for this entry in time + if entryCommit, ok := revs[entry.Name()]; ok { + commitsInfo[i].Commit = entryCommit + } else { + log.Debug("missing commit for %s", entry.Name()) + } + + // If the entry if a submodule add a submodule file for this + if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(commitsInfo[i].Commit, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile + } + } + + // Retrieve the commit for the treePath itself (see above). We basically + // get it for free during the tree traversal and it's used for listing + // pages to display information about newest commit for a given path. + var treeCommit *Commit + var ok bool + if treePath == "" { + treeCommit = commit + } else if treeCommit, ok = revs[""]; ok { + treeCommit.repo = commit.repo + } + return commitsInfo, treeCommit, nil +} + +func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { + var unHitEntryPaths []string + results := make(map[string]*Commit) + for _, p := range paths { + lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) + if err != nil { + return nil, nil, err + } + if lastCommit != nil { + results[p] = lastCommit + continue + } + + unHitEntryPaths = append(unHitEntryPaths, p) + } + + return results, unHitEntryPaths, nil +} + +// GetLastCommitForPaths returns last commit information +func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) { + // We read backwards from the commit to obtain all of the commits + revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...) + if err != nil { + return nil, err + } + + batchStdinWriter, batchReader, cancel := commit.repo.CatFileBatch(ctx) + defer cancel() + + commitsMap := map[string]*Commit{} + commitsMap[commit.ID.String()] = commit + + commitCommits := map[string]*Commit{} + for path, commitID := range revs { + c, ok := commitsMap[commitID] + if ok { + commitCommits[path] = c + continue + } + + if len(commitID) == 0 { + continue + } + + _, err := batchStdinWriter.Write([]byte(commitID + "\n")) + if err != nil { + return nil, err + } + _, typ, size, err := ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + if typ != "commit" { + if err := DiscardFull(batchReader, size+1); err != nil { + return nil, err + } + return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID) + } + c, err = CommitFromReader(commit.repo, MustIDFromString(commitID), io.LimitReader(batchReader, size)) + if err != nil { + return nil, err + } + if _, err := batchReader.Discard(1); err != nil { + return nil, err + } + commitCommits[path] = c + } + + return commitCommits, nil +} diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go deleted file mode 100644 index 31ffc9aec1..0000000000 --- a/modules/git/commit_info_gogit.go +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "context" - "path" - - "github.com/emirpasic/gods/trees/binaryheap" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" -) - -// GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { - entryPaths := make([]string, len(tes)+1) - // Get the commit for the treePath itself - entryPaths[0] = "" - for i, entry := range tes { - entryPaths[i+1] = entry.Name() - } - - commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex() - if commitGraphFile != nil { - defer commitGraphFile.Close() - } - - c, err := commitNodeIndex.Get(plumbing.Hash(commit.ID.RawValue())) - if err != nil { - return nil, nil, err - } - - var revs map[string]*Commit - if commit.repo.LastCommitCache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache) - if err != nil { - return nil, nil, err - } - if len(unHitPaths) > 0 { - revs2, err := GetLastCommitForPaths(ctx, commit.repo.LastCommitCache, c, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for k, v := range revs2 { - revs[k] = v - } - } - } else { - revs, err = GetLastCommitForPaths(ctx, nil, c, treePath, entryPaths) - } - if err != nil { - return nil, nil, err - } - - commit.repo.gogitStorage.Close() - - commitsInfo := make([]CommitInfo, len(tes)) - for i, entry := range tes { - commitsInfo[i] = CommitInfo{ - Entry: entry, - } - - // Check if we have found a commit for this entry in time - if entryCommit, ok := revs[entry.Name()]; ok { - commitsInfo[i].Commit = entryCommit - } - - // If the entry if a submodule add a submodule file for this - if entry.IsSubModule() { - subModuleURL := "" - var fullPath string - if len(treePath) > 0 { - fullPath = treePath + "/" + entry.Name() - } else { - fullPath = entry.Name() - } - if subModule, err := commit.GetSubModule(fullPath); err != nil { - return nil, nil, err - } else if subModule != nil { - subModuleURL = subModule.URL - } - subModuleFile := NewSubModuleFile(commitsInfo[i].Commit, subModuleURL, entry.ID.String()) - commitsInfo[i].SubModuleFile = subModuleFile - } - } - - // Retrieve the commit for the treePath itself (see above). We basically - // get it for free during the tree traversal and it's used for listing - // pages to display information about newest commit for a given path. - var treeCommit *Commit - var ok bool - if treePath == "" { - treeCommit = commit - } else if treeCommit, ok = revs[""]; ok { - treeCommit.repo = commit.repo - } - return commitsInfo, treeCommit, nil -} - -type commitAndPaths struct { - commit cgobject.CommitNode - // Paths that are still on the branch represented by commit - paths []string - // Set of hashes for the paths - hashes map[string]plumbing.Hash -} - -func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) { - tree, err := c.Tree() - if err != nil { - return nil, err - } - - // Optimize deep traversals by focusing only on the specific tree - if treePath != "" { - tree, err = tree.Tree(treePath) - if err != nil { - return nil, err - } - } - - return tree, nil -} - -func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) { - tree, err := getCommitTree(c, treePath) - if err == object.ErrDirectoryNotFound { - // The whole tree didn't exist, so return empty map - return make(map[string]plumbing.Hash), nil - } - if err != nil { - return nil, err - } - - hashes := make(map[string]plumbing.Hash) - for _, path := range paths { - if path != "" { - entry, err := tree.FindEntry(path) - if err == nil { - hashes[path] = entry.Hash - } - } else { - hashes[path] = tree.Hash - } - } - - return hashes, nil -} - -func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { - var unHitEntryPaths []string - results := make(map[string]*Commit) - for _, p := range paths { - lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) - if err != nil { - return nil, nil, err - } - if lastCommit != nil { - results[p] = lastCommit - continue - } - - unHitEntryPaths = append(unHitEntryPaths, p) - } - - return results, unHitEntryPaths, nil -} - -// GetLastCommitForPaths returns last commit information -func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*Commit, error) { - refSha := c.ID().String() - - // We do a tree traversal with nodes sorted by commit time - heap := binaryheap.NewWith(func(a, b any) int { - if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) { - return 1 - } - return -1 - }) - - resultNodes := make(map[string]cgobject.CommitNode) - initialHashes, err := getFileHashes(c, treePath, paths) - if err != nil { - return nil, err - } - - // Start search from the root commit and with full set of paths - heap.Push(&commitAndPaths{c, paths, initialHashes}) -heaploop: - for { - select { - case <-ctx.Done(): - if ctx.Err() == context.DeadlineExceeded { - break heaploop - } - return nil, ctx.Err() - default: - } - cIn, ok := heap.Pop() - if !ok { - break - } - current := cIn.(*commitAndPaths) - - // Load the parent commits for the one we are currently examining - numParents := current.commit.NumParents() - var parents []cgobject.CommitNode - for i := 0; i < numParents; i++ { - parent, err := current.commit.ParentNode(i) - if err != nil { - break - } - parents = append(parents, parent) - } - - // Examine the current commit and set of interesting paths - pathUnchanged := make([]bool, len(current.paths)) - parentHashes := make([]map[string]plumbing.Hash, len(parents)) - for j, parent := range parents { - parentHashes[j], err = getFileHashes(parent, treePath, current.paths) - if err != nil { - break - } - - for i, path := range current.paths { - if parentHashes[j][path] == current.hashes[path] { - pathUnchanged[i] = true - } - } - } - - var remainingPaths []string - for i, pth := range current.paths { - // The results could already contain some newer change for the same path, - // so don't override that and bail out on the file early. - if resultNodes[pth] == nil { - if pathUnchanged[i] { - // The path existed with the same hash in at least one parent so it could - // not have been changed in this commit directly. - remainingPaths = append(remainingPaths, pth) - } else { - // There are few possible cases how can we get here: - // - The path didn't exist in any parent, so it must have been created by - // this commit. - // - The path did exist in the parent commit, but the hash of the file has - // changed. - // - We are looking at a merge commit and the hash of the file doesn't - // match any of the hashes being merged. This is more common for directories, - // but it can also happen if a file is changed through conflict resolution. - resultNodes[pth] = current.commit - if err := cache.Put(refSha, path.Join(treePath, pth), current.commit.ID().String()); err != nil { - return nil, err - } - } - } - } - - if len(remainingPaths) > 0 { - // Add the parent nodes along with remaining paths to the heap for further - // processing. - for j, parent := range parents { - // Combine remainingPath with paths available on the parent branch - // and make union of them - remainingPathsForParent := make([]string, 0, len(remainingPaths)) - newRemainingPaths := make([]string, 0, len(remainingPaths)) - for _, path := range remainingPaths { - if parentHashes[j][path] == current.hashes[path] { - remainingPathsForParent = append(remainingPathsForParent, path) - } else { - newRemainingPaths = append(newRemainingPaths, path) - } - } - - if remainingPathsForParent != nil { - heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]}) - } - - if len(newRemainingPaths) == 0 { - break - } else { - remainingPaths = newRemainingPaths - } - } - } - } - - // Post-processing - result := make(map[string]*Commit) - for path, commitNode := range resultNodes { - commit, err := commitNode.Commit() - if err != nil { - return nil, err - } - result[path] = convertCommit(commit) - } - - return result, nil -} diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go deleted file mode 100644 index 7c369b07f9..0000000000 --- a/modules/git/commit_info_nogogit.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "context" - "fmt" - "io" - "path" - "sort" - - "code.gitea.io/gitea/modules/log" -) - -// GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath string) ([]CommitInfo, *Commit, error) { - entryPaths := make([]string, len(tes)+1) - // Get the commit for the treePath itself - entryPaths[0] = "" - for i, entry := range tes { - entryPaths[i+1] = entry.Name() - } - - var err error - - var revs map[string]*Commit - if commit.repo.LastCommitCache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache) - if err != nil { - return nil, nil, err - } - if len(unHitPaths) > 0 { - sort.Strings(unHitPaths) - commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for pth, found := range commits { - revs[pth] = found - } - } - } else { - sort.Strings(entryPaths) - revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths) - } - if err != nil { - return nil, nil, err - } - - commitsInfo := make([]CommitInfo, len(tes)) - for i, entry := range tes { - commitsInfo[i] = CommitInfo{ - Entry: entry, - } - - // Check if we have found a commit for this entry in time - if entryCommit, ok := revs[entry.Name()]; ok { - commitsInfo[i].Commit = entryCommit - } else { - log.Debug("missing commit for %s", entry.Name()) - } - - // If the entry if a submodule add a submodule file for this - if entry.IsSubModule() { - subModuleURL := "" - var fullPath string - if len(treePath) > 0 { - fullPath = treePath + "/" + entry.Name() - } else { - fullPath = entry.Name() - } - if subModule, err := commit.GetSubModule(fullPath); err != nil { - return nil, nil, err - } else if subModule != nil { - subModuleURL = subModule.URL - } - subModuleFile := NewSubModuleFile(commitsInfo[i].Commit, subModuleURL, entry.ID.String()) - commitsInfo[i].SubModuleFile = subModuleFile - } - } - - // Retrieve the commit for the treePath itself (see above). We basically - // get it for free during the tree traversal and it's used for listing - // pages to display information about newest commit for a given path. - var treeCommit *Commit - var ok bool - if treePath == "" { - treeCommit = commit - } else if treeCommit, ok = revs[""]; ok { - treeCommit.repo = commit.repo - } - return commitsInfo, treeCommit, nil -} - -func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { - var unHitEntryPaths []string - results := make(map[string]*Commit) - for _, p := range paths { - lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) - if err != nil { - return nil, nil, err - } - if lastCommit != nil { - results[p] = lastCommit - continue - } - - unHitEntryPaths = append(unHitEntryPaths, p) - } - - return results, unHitEntryPaths, nil -} - -// GetLastCommitForPaths returns last commit information -func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) { - // We read backwards from the commit to obtain all of the commits - revs, err := WalkGitLog(ctx, commit.repo, commit, treePath, paths...) - if err != nil { - return nil, err - } - - batchStdinWriter, batchReader, cancel := commit.repo.CatFileBatch(ctx) - defer cancel() - - commitsMap := map[string]*Commit{} - commitsMap[commit.ID.String()] = commit - - commitCommits := map[string]*Commit{} - for path, commitID := range revs { - c, ok := commitsMap[commitID] - if ok { - commitCommits[path] = c - continue - } - - if len(commitID) == 0 { - continue - } - - _, err := batchStdinWriter.Write([]byte(commitID + "\n")) - if err != nil { - return nil, err - } - _, typ, size, err := ReadBatchLine(batchReader) - if err != nil { - return nil, err - } - if typ != "commit" { - if err := DiscardFull(batchReader, size+1); err != nil { - return nil, err - } - return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID) - } - c, err = CommitFromReader(commit.repo, MustIDFromString(commitID), io.LimitReader(batchReader, size)) - if err != nil { - return nil, err - } - if _, err := batchReader.Discard(1); err != nil { - return nil, err - } - commitCommits[path] = c - } - - return commitCommits, nil -} diff --git a/modules/git/commit_sha256_test.go b/modules/git/commit_sha256_test.go index f0e392a35e..9e56829f45 100644 --- a/modules/git/commit_sha256_test.go +++ b/modules/git/commit_sha256_test.go @@ -1,8 +1,6 @@ // Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build !gogit - package git import ( diff --git a/modules/git/git.go b/modules/git/git.go index 70232c86a0..d1e841eeb8 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -186,12 +186,12 @@ func InitFull(ctx context.Context) (err error) { globalCommandArgs = append(globalCommandArgs, "-c", "credential.helper=") } SupportProcReceive = CheckGitVersionAtLeast("2.29") == nil - SupportHashSha256 = CheckGitVersionAtLeast("2.42") == nil && !isGogit + SupportHashSha256 = CheckGitVersionAtLeast("2.42") == nil SupportCheckAttrOnBare = CheckGitVersionAtLeast("2.40") == nil if SupportHashSha256 { SupportedObjectFormats = append(SupportedObjectFormats, Sha256ObjectFormat) } else { - log.Warn("sha256 hash support is disabled - requires Git >= 2.42. Gogit is currently unsupported") + log.Warn("sha256 hash support is disabled - requires Git >= 2.42") } InvertedGitFlushEnv = CheckGitVersionEqual("2.43.1") == nil diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index 5b62b90b27..8c7ee5a933 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -4,6 +4,7 @@ package git import ( + "context" "crypto/sha256" "fmt" @@ -112,3 +113,47 @@ func (c *LastCommitCache) GetCommitByPath(commitID, entryPath string) (*Commit, return lastCommit, nil } + +// CacheCommit will cache the commit from the gitRepository +func (c *Commit) CacheCommit(ctx context.Context) error { + if c.repo.LastCommitCache == nil { + return nil + } + return c.recursiveCache(ctx, &c.Tree, "", 1) +} + +func (c *Commit) recursiveCache(ctx context.Context, tree *Tree, treePath string, level int) error { + if level == 0 { + return nil + } + + entries, err := tree.ListEntries() + if err != nil { + return err + } + + entryPaths := make([]string, len(entries)) + for i, entry := range entries { + entryPaths[i] = entry.Name() + } + + _, err = WalkGitLog(ctx, c.repo, c, treePath, entryPaths...) + if err != nil { + return err + } + + for _, treeEntry := range entries { + // entryMap won't contain "" therefore skip this. + if treeEntry.IsDir() { + subTree, err := tree.SubTree(treeEntry.Name()) + if err != nil { + return err + } + if err := c.recursiveCache(ctx, subTree, treeEntry.Name(), level-1); err != nil { + return err + } + } + } + + return nil +} diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go deleted file mode 100644 index 3afc213094..0000000000 --- a/modules/git/last_commit_cache_gogit.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "context" - - "github.com/go-git/go-git/v5/plumbing" - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" -) - -// CacheCommit will cache the commit from the gitRepository -func (c *Commit) CacheCommit(ctx context.Context) error { - if c.repo.LastCommitCache == nil { - return nil - } - commitNodeIndex, _ := c.repo.CommitNodeIndex() - - index, err := commitNodeIndex.Get(plumbing.Hash(c.ID.RawValue())) - if err != nil { - return err - } - - return c.recursiveCache(ctx, index, &c.Tree, "", 1) -} - -func (c *Commit) recursiveCache(ctx context.Context, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { - if level == 0 { - return nil - } - - entries, err := tree.ListEntries() - if err != nil { - return err - } - - entryPaths := make([]string, len(entries)) - entryMap := make(map[string]*TreeEntry) - for i, entry := range entries { - entryPaths[i] = entry.Name() - entryMap[entry.Name()] = entry - } - - commits, err := GetLastCommitForPaths(ctx, c.repo.LastCommitCache, index, treePath, entryPaths) - if err != nil { - return err - } - - for entry := range commits { - if entryMap[entry].IsDir() { - subTree, err := tree.SubTree(entry) - if err != nil { - return err - } - if err := c.recursiveCache(ctx, index, subTree, entry, level-1); err != nil { - return err - } - } - } - - return nil -} diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go deleted file mode 100644 index 155cb3cb7c..0000000000 --- a/modules/git/last_commit_cache_nogogit.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "context" -) - -// CacheCommit will cache the commit from the gitRepository -func (c *Commit) CacheCommit(ctx context.Context) error { - if c.repo.LastCommitCache == nil { - return nil - } - return c.recursiveCache(ctx, &c.Tree, "", 1) -} - -func (c *Commit) recursiveCache(ctx context.Context, tree *Tree, treePath string, level int) error { - if level == 0 { - return nil - } - - entries, err := tree.ListEntries() - if err != nil { - return err - } - - entryPaths := make([]string, len(entries)) - for i, entry := range entries { - entryPaths[i] = entry.Name() - } - - _, err = WalkGitLog(ctx, c.repo, c, treePath, entryPaths...) - if err != nil { - return err - } - - for _, treeEntry := range entries { - // entryMap won't contain "" therefore skip this. - if treeEntry.IsDir() { - subTree, err := tree.SubTree(treeEntry.Name()) - if err != nil { - return err - } - if err := c.recursiveCache(ctx, subTree, treeEntry.Name(), level-1); err != nil { - return err - } - } - } - - return nil -} diff --git a/modules/git/notes.go b/modules/git/notes.go index 63539cb3a2..ee628c0436 100644 --- a/modules/git/notes.go +++ b/modules/git/notes.go @@ -3,6 +3,14 @@ package git +import ( + "context" + "io" + "strings" + + "code.gitea.io/gitea/modules/log" +) + // NotesRef is the git ref where Gitea will look for git-notes data. // The value ("refs/notes/commits") is the default ref used by git-notes. const NotesRef = "refs/notes/commits" @@ -12,3 +20,80 @@ type Note struct { Message []byte Commit *Commit } + +// GetNote retrieves the git-notes data for a given commit. +// FIXME: Add LastCommitCache support +func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error { + log.Trace("Searching for git note corresponding to the commit %q in the repository %q", commitID, repo.Path) + notes, err := repo.GetCommit(NotesRef) + if err != nil { + if IsErrNotExist(err) { + return err + } + log.Error("Unable to get commit from ref %q. Error: %v", NotesRef, err) + return err + } + + path := "" + + tree := ¬es.Tree + log.Trace("Found tree with ID %q while searching for git note corresponding to the commit %q", tree.ID, commitID) + + var entry *TreeEntry + originalCommitID := commitID + for len(commitID) > 2 { + entry, err = tree.GetTreeEntryByPath(commitID) + if err == nil { + path += commitID + break + } + if IsErrNotExist(err) { + tree, err = tree.SubTree(commitID[0:2]) + path += commitID[0:2] + "/" + commitID = commitID[2:] + } + if err != nil { + // Err may have been updated by the SubTree we need to recheck if it's again an ErrNotExist + if !IsErrNotExist(err) { + log.Error("Unable to find git note corresponding to the commit %q. Error: %v", originalCommitID, err) + } + return err + } + } + + blob := entry.Blob() + dataRc, err := blob.DataAsync() + if err != nil { + log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err) + return err + } + closed := false + defer func() { + if !closed { + _ = dataRc.Close() + } + }() + d, err := io.ReadAll(dataRc) + if err != nil { + log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err) + return err + } + _ = dataRc.Close() + closed = true + note.Message = d + + treePath := "" + if idx := strings.LastIndex(path, "/"); idx > -1 { + treePath = path[:idx] + path = path[idx+1:] + } + + lastCommits, err := GetLastCommitForPaths(ctx, notes, treePath, []string{path}) + if err != nil { + log.Error("Unable to get the commit for the path %q. Error: %v", treePath, err) + return err + } + note.Commit = lastCommits[path] + + return nil +} diff --git a/modules/git/notes_gogit.go b/modules/git/notes_gogit.go deleted file mode 100644 index f802443b00..0000000000 --- a/modules/git/notes_gogit.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "context" - "io" - - "code.gitea.io/gitea/modules/log" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// GetNote retrieves the git-notes data for a given commit. -// FIXME: Add LastCommitCache support -func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error { - log.Trace("Searching for git note corresponding to the commit %q in the repository %q", commitID, repo.Path) - notes, err := repo.GetCommit(NotesRef) - if err != nil { - if IsErrNotExist(err) { - return err - } - log.Error("Unable to get commit from ref %q. Error: %v", NotesRef, err) - return err - } - - remainingCommitID := commitID - path := "" - currentTree := notes.Tree.gogitTree - log.Trace("Found tree with ID %q while searching for git note corresponding to the commit %q", currentTree.Entries[0].Name, commitID) - var file *object.File - for len(remainingCommitID) > 2 { - file, err = currentTree.File(remainingCommitID) - if err == nil { - path += remainingCommitID - break - } - if err == object.ErrFileNotFound { - currentTree, err = currentTree.Tree(remainingCommitID[0:2]) - path += remainingCommitID[0:2] + "/" - remainingCommitID = remainingCommitID[2:] - } - if err != nil { - if err == object.ErrDirectoryNotFound { - return ErrNotExist{ID: remainingCommitID, RelPath: path} - } - log.Error("Unable to find git note corresponding to the commit %q. Error: %v", commitID, err) - return err - } - } - - blob := file.Blob - dataRc, err := blob.Reader() - if err != nil { - log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err) - return err - } - - defer dataRc.Close() - d, err := io.ReadAll(dataRc) - if err != nil { - log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err) - return err - } - note.Message = d - - commitNodeIndex, commitGraphFile := repo.CommitNodeIndex() - if commitGraphFile != nil { - defer commitGraphFile.Close() - } - - commitNode, err := commitNodeIndex.Get(plumbing.Hash(notes.ID.RawValue())) - if err != nil { - return err - } - - lastCommits, err := GetLastCommitForPaths(ctx, nil, commitNode, "", []string{path}) - if err != nil { - log.Error("Unable to get the commit for the path %q. Error: %v", path, err) - return err - } - note.Commit = lastCommits[path] - - return nil -} diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go deleted file mode 100644 index 4da375c321..0000000000 --- a/modules/git/notes_nogogit.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "context" - "io" - "strings" - - "code.gitea.io/gitea/modules/log" -) - -// GetNote retrieves the git-notes data for a given commit. -// FIXME: Add LastCommitCache support -func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) error { - log.Trace("Searching for git note corresponding to the commit %q in the repository %q", commitID, repo.Path) - notes, err := repo.GetCommit(NotesRef) - if err != nil { - if IsErrNotExist(err) { - return err - } - log.Error("Unable to get commit from ref %q. Error: %v", NotesRef, err) - return err - } - - path := "" - - tree := ¬es.Tree - log.Trace("Found tree with ID %q while searching for git note corresponding to the commit %q", tree.ID, commitID) - - var entry *TreeEntry - originalCommitID := commitID - for len(commitID) > 2 { - entry, err = tree.GetTreeEntryByPath(commitID) - if err == nil { - path += commitID - break - } - if IsErrNotExist(err) { - tree, err = tree.SubTree(commitID[0:2]) - path += commitID[0:2] + "/" - commitID = commitID[2:] - } - if err != nil { - // Err may have been updated by the SubTree we need to recheck if it's again an ErrNotExist - if !IsErrNotExist(err) { - log.Error("Unable to find git note corresponding to the commit %q. Error: %v", originalCommitID, err) - } - return err - } - } - - blob := entry.Blob() - dataRc, err := blob.DataAsync() - if err != nil { - log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err) - return err - } - closed := false - defer func() { - if !closed { - _ = dataRc.Close() - } - }() - d, err := io.ReadAll(dataRc) - if err != nil { - log.Error("Unable to read blob with ID %q. Error: %v", blob.ID, err) - return err - } - _ = dataRc.Close() - closed = true - note.Message = d - - treePath := "" - if idx := strings.LastIndex(path, "/"); idx > -1 { - treePath = path[:idx] - path = path[idx+1:] - } - - lastCommits, err := GetLastCommitForPaths(ctx, notes, treePath, []string{path}) - if err != nil { - log.Error("Unable to get the commit for the path %q. Error: %v", treePath, err) - return err - } - note.Commit = lastCommits[path] - - return nil -} diff --git a/modules/git/object_id_gogit.go b/modules/git/object_id_gogit.go deleted file mode 100644 index db4c4ae0bd..0000000000 --- a/modules/git/object_id_gogit.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT -//go:build gogit - -package git - -import ( - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/hash" -) - -func ParseGogitHash(h plumbing.Hash) ObjectID { - switch hash.Size { - case 20: - return Sha1ObjectFormat.MustID(h[:]) - case 32: - return Sha256ObjectFormat.MustID(h[:]) - } - - return nil -} - -func ParseGogitHashArray(objectIDs []plumbing.Hash) []ObjectID { - ret := make([]ObjectID, len(objectIDs)) - for i, h := range objectIDs { - ret[i] = ParseGogitHash(h) - } - - return ret -} diff --git a/modules/git/parse_nogogit.go b/modules/git/parse.go similarity index 99% rename from modules/git/parse_nogogit.go rename to modules/git/parse.go index 546b38be37..8c2c411db6 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse.go @@ -1,8 +1,6 @@ // Copyright 2018 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build !gogit - package git import ( diff --git a/modules/git/parse_gogit.go b/modules/git/parse_gogit.go deleted file mode 100644 index 74d258de8e..0000000000 --- a/modules/git/parse_gogit.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "bytes" - "fmt" - "strconv" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/hash" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// ParseTreeEntries parses the output of a `git ls-tree -l` command. -func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { - return parseTreeEntries(data, nil) -} - -func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { - entries := make([]*TreeEntry, 0, 10) - for pos := 0; pos < len(data); { - // expect line to be of the form " \t" - entry := new(TreeEntry) - entry.gogitTreeEntry = &object.TreeEntry{} - entry.ptree = ptree - if pos+6 > len(data) { - return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) - } - switch string(data[pos : pos+6]) { - case "100644": - entry.gogitTreeEntry.Mode = filemode.Regular - pos += 12 // skip over "100644 blob " - case "100755": - entry.gogitTreeEntry.Mode = filemode.Executable - pos += 12 // skip over "100755 blob " - case "120000": - entry.gogitTreeEntry.Mode = filemode.Symlink - pos += 12 // skip over "120000 blob " - case "160000": - entry.gogitTreeEntry.Mode = filemode.Submodule - pos += 14 // skip over "160000 object " - case "040000": - entry.gogitTreeEntry.Mode = filemode.Dir - pos += 12 // skip over "040000 tree " - default: - return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6])) - } - - // in hex format, not byte format .... - if pos+hash.Size*2 > len(data) { - return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) - } - var err error - entry.ID, err = NewIDFromString(string(data[pos : pos+hash.Size*2])) - if err != nil { - return nil, fmt.Errorf("invalid ls-tree output: %w", err) - } - entry.gogitTreeEntry.Hash = plumbing.Hash(entry.ID.RawValue()) - pos += 41 // skip over sha and trailing space - - end := pos + bytes.IndexByte(data[pos:], '\t') - if end < pos { - return nil, fmt.Errorf("Invalid ls-tree -l output: %s", string(data)) - } - entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(data[pos:end])), 10, 64) - entry.sized = true - - pos = end + 1 - - end = pos + bytes.IndexByte(data[pos:], '\n') - if end < pos { - return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) - } - - // In case entry name is surrounded by double quotes(it happens only in git-shell). - if data[pos] == '"' { - var err error - entry.gogitTreeEntry.Name, err = strconv.Unquote(string(data[pos:end])) - if err != nil { - return nil, fmt.Errorf("Invalid ls-tree output: %w", err) - } - } else { - entry.gogitTreeEntry.Name = string(data[pos:end]) - } - - pos = end + 1 - entries = append(entries, entry) - } - return entries, nil -} diff --git a/modules/git/parse_gogit_test.go b/modules/git/parse_gogit_test.go deleted file mode 100644 index 7622478550..0000000000 --- a/modules/git/parse_gogit_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "fmt" - "testing" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestParseTreeEntries(t *testing.T) { - testCases := []struct { - Input string - Expected []*TreeEntry - }{ - { - Input: "", - Expected: []*TreeEntry{}, - }, - { - Input: "100644 blob 61ab7345a1a3bbc590068ccae37b8515cfc5843c 1022\texample/file2.txt\n", - Expected: []*TreeEntry{ - { - ID: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), - gogitTreeEntry: &object.TreeEntry{ - Hash: plumbing.Hash(MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c").RawValue()), - Name: "example/file2.txt", - Mode: filemode.Regular, - }, - size: 1022, - sized: true, - }, - }, - }, - { - Input: "120000 blob 61ab7345a1a3bbc590068ccae37b8515cfc5843c 234131\t\"example/\\n.txt\"\n" + - "040000 tree 1d01fb729fb0db5881daaa6030f9f2d3cd3d5ae8 -\texample\n", - Expected: []*TreeEntry{ - { - ID: MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c"), - gogitTreeEntry: &object.TreeEntry{ - Hash: plumbing.Hash(MustIDFromString("61ab7345a1a3bbc590068ccae37b8515cfc5843c").RawValue()), - Name: "example/\n.txt", - Mode: filemode.Symlink, - }, - size: 234131, - sized: true, - }, - { - ID: MustIDFromString("1d01fb729fb0db5881daaa6030f9f2d3cd3d5ae8"), - sized: true, - gogitTreeEntry: &object.TreeEntry{ - Hash: plumbing.Hash(MustIDFromString("1d01fb729fb0db5881daaa6030f9f2d3cd3d5ae8").RawValue()), - Name: "example", - Mode: filemode.Dir, - }, - }, - }, - }, - } - - for _, testCase := range testCases { - entries, err := ParseTreeEntries([]byte(testCase.Input)) - require.NoError(t, err) - if len(entries) > 1 { - fmt.Println(testCase.Expected[0].ID) - fmt.Println(entries[0].ID) - } - assert.EqualValues(t, testCase.Expected, entries) - } -} diff --git a/modules/git/parse_nogogit_test.go b/modules/git/parse_test.go similarity index 99% rename from modules/git/parse_nogogit_test.go rename to modules/git/parse_test.go index cc1d02cc0c..89c6e0399b 100644 --- a/modules/git/parse_nogogit_test.go +++ b/modules/git/parse_test.go @@ -1,8 +1,6 @@ // Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build !gogit - package git import ( diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs.go similarity index 90% rename from modules/git/pipeline/lfs_nogogit.go rename to modules/git/pipeline/lfs.go index 349cfbd9ce..55c49aaf3d 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs.go @@ -1,21 +1,42 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build !gogit - package pipeline import ( "bufio" "bytes" + "fmt" "io" "sort" "strings" "sync" + "time" "code.gitea.io/gitea/modules/git" ) +// LFSResult represents commits found using a provided pointer file hash +type LFSResult struct { + Name string + SHA string + Summary string + When time.Time + ParentHashes []git.ObjectID + BranchName string + FullCommitName string +} + +type lfsResultSlice []*LFSResult + +func (a lfsResultSlice) Len() int { return len(a) } +func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } + +func lfsError(msg string, err error) error { + return fmt.Errorf("LFS error occurred, %s: err: %w", msg, err) +} + // FindLFSFile finds commits that contain a provided pointer file hash func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, error) { resultsMap := map[string]*LFSResult{} diff --git a/modules/git/pipeline/lfs_common.go b/modules/git/pipeline/lfs_common.go deleted file mode 100644 index 188e7d4d65..0000000000 --- a/modules/git/pipeline/lfs_common.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package pipeline - -import ( - "fmt" - "time" - - "code.gitea.io/gitea/modules/git" -) - -// LFSResult represents commits found using a provided pointer file hash -type LFSResult struct { - Name string - SHA string - Summary string - When time.Time - ParentHashes []git.ObjectID - BranchName string - FullCommitName string -} - -type lfsResultSlice []*LFSResult - -func (a lfsResultSlice) Len() int { return len(a) } -func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } - -func lfsError(msg string, err error) error { - return fmt.Errorf("LFS error occurred, %s: err: %w", msg, err) -} diff --git a/modules/git/pipeline/lfs_gogit.go b/modules/git/pipeline/lfs_gogit.go deleted file mode 100644 index adcf8ed09c..0000000000 --- a/modules/git/pipeline/lfs_gogit.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package pipeline - -import ( - "bufio" - "io" - "sort" - "strings" - "sync" - - "code.gitea.io/gitea/modules/git" - - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// FindLFSFile finds commits that contain a provided pointer file hash -func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, error) { - resultsMap := map[string]*LFSResult{} - results := make([]*LFSResult, 0) - - basePath := repo.Path - gogitRepo := repo.GoGitRepo() - - commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ - Order: gogit.LogOrderCommitterTime, - All: true, - }) - if err != nil { - return nil, lfsError("failed to get GoGit CommitsIter", err) - } - - err = commitsIter.ForEach(func(gitCommit *object.Commit) error { - tree, err := gitCommit.Tree() - if err != nil { - return err - } - treeWalker := object.NewTreeWalker(tree, true, nil) - defer treeWalker.Close() - for { - name, entry, err := treeWalker.Next() - if err == io.EOF { - break - } - if entry.Hash == plumbing.Hash(objectID.RawValue()) { - parents := make([]git.ObjectID, len(gitCommit.ParentHashes)) - for i, parentCommitID := range gitCommit.ParentHashes { - parents[i] = git.ParseGogitHash(parentCommitID) - } - - result := LFSResult{ - Name: name, - SHA: gitCommit.Hash.String(), - Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], - When: gitCommit.Author.When, - ParentHashes: parents, - } - resultsMap[gitCommit.Hash.String()+":"+name] = &result - } - } - return nil - }) - if err != nil && err != io.EOF { - return nil, lfsError("failure in CommitIter.ForEach", err) - } - - for _, result := range resultsMap { - hasParent := false - for _, parentHash := range result.ParentHashes { - if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { - break - } - } - if !hasParent { - results = append(results, result) - } - } - - sort.Sort(lfsResultSlice(results)) - - // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple - shasToNameReader, shasToNameWriter := io.Pipe() - nameRevStdinReader, nameRevStdinWriter := io.Pipe() - errChan := make(chan error, 1) - wg := sync.WaitGroup{} - wg.Add(3) - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(nameRevStdinReader) - i := 0 - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - result := results[i] - result.FullCommitName = line - result.BranchName = strings.Split(line, "~")[0] - i++ - } - }() - go NameRevStdin(repo.Ctx, shasToNameReader, nameRevStdinWriter, &wg, basePath) - go func() { - defer wg.Done() - defer shasToNameWriter.Close() - for _, result := range results { - i := 0 - if i < len(result.SHA) { - n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) - if err != nil { - errChan <- err - break - } - i += n - } - n := 0 - for n < 1 { - n, err = shasToNameWriter.Write([]byte{'\n'}) - if err != nil { - errChan <- err - break - } - - } - - } - }() - - wg.Wait() - - select { - case err, has := <-errChan: - if has { - return nil, lfsError("unable to obtain name for LFS files", err) - } - default: - } - - return results, nil -} diff --git a/modules/git/repo_base.go b/modules/git/repo_base.go index 6c148d9af5..8c34efc2c7 100644 --- a/modules/git/repo_base.go +++ b/modules/git/repo_base.go @@ -1,6 +1,116 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package git -var isGogit bool +import ( + "bufio" + "context" + "errors" + "path/filepath" + + "code.gitea.io/gitea/modules/log" +) + +// Repository represents a Git repository. +type Repository struct { + Path string + + tagCache *ObjectCache + + gpgSettings *GPGSettings + + batchInUse bool + batchCancel context.CancelFunc + batchReader *bufio.Reader + batchWriter WriteCloserError + + checkInUse bool + checkCancel context.CancelFunc + checkReader *bufio.Reader + checkWriter WriteCloserError + + Ctx context.Context + LastCommitCache *LastCommitCache + + objectFormat ObjectFormat +} + +// openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext. +func openRepositoryWithDefaultContext(repoPath string) (*Repository, error) { + return OpenRepository(DefaultContext, repoPath) +} + +// OpenRepository opens the repository at the given path with the provided context. +func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { + repoPath, err := filepath.Abs(repoPath) + if err != nil { + return nil, err + } else if !isDir(repoPath) { + return nil, errors.New("no such file or directory") + } + + // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! + if err := EnsureValidGitRepository(ctx, repoPath); err != nil { + return nil, err + } + + repo := &Repository{ + Path: repoPath, + tagCache: newObjectCache(), + Ctx: ctx, + } + + repo.batchWriter, repo.batchReader, repo.batchCancel = CatFileBatch(ctx, repoPath) + repo.checkWriter, repo.checkReader, repo.checkCancel = CatFileBatchCheck(ctx, repoPath) + + return repo, nil +} + +// CatFileBatch obtains a CatFileBatch for this repository +func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func()) { + if repo.batchCancel == nil || repo.batchInUse { + log.Debug("Opening temporary cat file batch for: %s", repo.Path) + return CatFileBatch(ctx, repo.Path) + } + repo.batchInUse = true + return repo.batchWriter, repo.batchReader, func() { + repo.batchInUse = false + } +} + +// CatFileBatchCheck obtains a CatFileBatchCheck for this repository +func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func()) { + if repo.checkCancel == nil || repo.checkInUse { + log.Debug("Opening temporary cat file batch-check for: %s", repo.Path) + return CatFileBatchCheck(ctx, repo.Path) + } + repo.checkInUse = true + return repo.checkWriter, repo.checkReader, func() { + repo.checkInUse = false + } +} + +func (repo *Repository) Close() error { + if repo == nil { + return nil + } + if repo.batchCancel != nil { + repo.batchCancel() + repo.batchReader = nil + repo.batchWriter = nil + repo.batchCancel = nil + repo.batchInUse = false + } + if repo.checkCancel != nil { + repo.checkCancel() + repo.checkCancel = nil + repo.checkReader = nil + repo.checkWriter = nil + repo.checkInUse = false + } + repo.LastCommitCache = nil + repo.tagCache = nil + return nil +} diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go deleted file mode 100644 index 3ca5eb36c6..0000000000 --- a/modules/git/repo_base_gogit.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "context" - "errors" - "path/filepath" - - gitealog "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/osfs" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/storage/filesystem" -) - -func init() { - isGogit = true -} - -// Repository represents a Git repository. -type Repository struct { - Path string - - tagCache *ObjectCache - - gogitRepo *gogit.Repository - gogitStorage *filesystem.Storage - gpgSettings *GPGSettings - - Ctx context.Context - LastCommitCache *LastCommitCache - objectFormat ObjectFormat -} - -// openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext. -func openRepositoryWithDefaultContext(repoPath string) (*Repository, error) { - return OpenRepository(DefaultContext, repoPath) -} - -// OpenRepository opens the repository at the given path within the context.Context -func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { - repoPath, err := filepath.Abs(repoPath) - if err != nil { - return nil, err - } else if !isDir(repoPath) { - return nil, errors.New("no such file or directory") - } - - fs := osfs.New(repoPath) - _, err = fs.Stat(".git") - if err == nil { - fs, err = fs.Chroot(".git") - if err != nil { - return nil, err - } - } - // the "clone --shared" repo doesn't work well with go-git AlternativeFS, https://github.com/go-git/go-git/issues/1006 - // so use "/" for AlternatesFS, I guess it is the same behavior as current nogogit (no limitation or check for the "objects/info/alternates" paths), trust the "clone" command executed by the server. - var altFs billy.Filesystem - if setting.IsWindows { - altFs = osfs.New(filepath.VolumeName(setting.RepoRootPath) + "\\") // TODO: does it really work for Windows? Need some time to check. - } else { - altFs = osfs.New("/") - } - storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true, LargeObjectThreshold: setting.Git.LargeObjectThreshold, AlternatesFS: altFs}) - gogitRepo, err := gogit.Open(storage, fs) - if err != nil { - return nil, err - } - - return &Repository{ - Path: repoPath, - gogitRepo: gogitRepo, - gogitStorage: storage, - tagCache: newObjectCache(), - Ctx: ctx, - objectFormat: ParseGogitHash(plumbing.ZeroHash).Type(), - }, nil -} - -// Close this repository, in particular close the underlying gogitStorage if this is not nil -func (repo *Repository) Close() error { - if repo == nil || repo.gogitStorage == nil { - return nil - } - if err := repo.gogitStorage.Close(); err != nil { - gitealog.Error("Error closing storage: %v", err) - } - repo.gogitStorage = nil - repo.LastCommitCache = nil - repo.tagCache = nil - return nil -} - -// GoGitRepo gets the go-git repo representation -func (repo *Repository) GoGitRepo() *gogit.Repository { - return repo.gogitRepo -} diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go deleted file mode 100644 index 50a0a975b8..0000000000 --- a/modules/git/repo_base_nogogit.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "bufio" - "context" - "errors" - "path/filepath" - - "code.gitea.io/gitea/modules/log" -) - -func init() { - isGogit = false -} - -// Repository represents a Git repository. -type Repository struct { - Path string - - tagCache *ObjectCache - - gpgSettings *GPGSettings - - batchInUse bool - batchCancel context.CancelFunc - batchReader *bufio.Reader - batchWriter WriteCloserError - - checkInUse bool - checkCancel context.CancelFunc - checkReader *bufio.Reader - checkWriter WriteCloserError - - Ctx context.Context - LastCommitCache *LastCommitCache - - objectFormat ObjectFormat -} - -// openRepositoryWithDefaultContext opens the repository at the given path with DefaultContext. -func openRepositoryWithDefaultContext(repoPath string) (*Repository, error) { - return OpenRepository(DefaultContext, repoPath) -} - -// OpenRepository opens the repository at the given path with the provided context. -func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) { - repoPath, err := filepath.Abs(repoPath) - if err != nil { - return nil, err - } else if !isDir(repoPath) { - return nil, errors.New("no such file or directory") - } - - // Now because of some insanity with git cat-file not immediately failing if not run in a valid git directory we need to run git rev-parse first! - if err := EnsureValidGitRepository(ctx, repoPath); err != nil { - return nil, err - } - - repo := &Repository{ - Path: repoPath, - tagCache: newObjectCache(), - Ctx: ctx, - } - - repo.batchWriter, repo.batchReader, repo.batchCancel = CatFileBatch(ctx, repoPath) - repo.checkWriter, repo.checkReader, repo.checkCancel = CatFileBatchCheck(ctx, repoPath) - - return repo, nil -} - -// CatFileBatch obtains a CatFileBatch for this repository -func (repo *Repository) CatFileBatch(ctx context.Context) (WriteCloserError, *bufio.Reader, func()) { - if repo.batchCancel == nil || repo.batchInUse { - log.Debug("Opening temporary cat file batch for: %s", repo.Path) - return CatFileBatch(ctx, repo.Path) - } - repo.batchInUse = true - return repo.batchWriter, repo.batchReader, func() { - repo.batchInUse = false - } -} - -// CatFileBatchCheck obtains a CatFileBatchCheck for this repository -func (repo *Repository) CatFileBatchCheck(ctx context.Context) (WriteCloserError, *bufio.Reader, func()) { - if repo.checkCancel == nil || repo.checkInUse { - log.Debug("Opening temporary cat file batch-check for: %s", repo.Path) - return CatFileBatchCheck(ctx, repo.Path) - } - repo.checkInUse = true - return repo.checkWriter, repo.checkReader, func() { - repo.checkInUse = false - } -} - -func (repo *Repository) Close() error { - if repo == nil { - return nil - } - if repo.batchCancel != nil { - repo.batchCancel() - repo.batchReader = nil - repo.batchWriter = nil - repo.batchCancel = nil - repo.batchInUse = false - } - if repo.checkCancel != nil { - repo.checkCancel() - repo.checkCancel = nil - repo.checkReader = nil - repo.checkWriter = nil - repo.checkInUse = false - } - repo.LastCommitCache = nil - repo.tagCache = nil - return nil -} diff --git a/modules/git/repo_blob.go b/modules/git/repo_blob.go deleted file mode 100644 index 698b6c7074..0000000000 --- a/modules/git/repo_blob.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package git - -// GetBlob finds the blob object in the repository. -func (repo *Repository) GetBlob(idStr string) (*Blob, error) { - id, err := NewIDFromString(idStr) - if err != nil { - return nil, err - } - return repo.getBlob(id) -} diff --git a/modules/git/repo_blob_gogit.go b/modules/git/repo_blob_gogit.go deleted file mode 100644 index 66c8c2775c..0000000000 --- a/modules/git/repo_blob_gogit.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "github.com/go-git/go-git/v5/plumbing" -) - -func (repo *Repository) getBlob(id ObjectID) (*Blob, error) { - encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, plumbing.Hash(id.RawValue())) - if err != nil { - return nil, ErrNotExist{id.String(), ""} - } - - return &Blob{ - ID: id, - gogitEncodedObj: encodedObj, - }, nil -} diff --git a/modules/git/repo_blob_nogogit.go b/modules/git/repo_blob_nogogit.go deleted file mode 100644 index 04b0fb00ff..0000000000 --- a/modules/git/repo_blob_nogogit.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -func (repo *Repository) getBlob(id ObjectID) (*Blob, error) { - if id.IsZero() { - return nil, ErrNotExist{id.String(), ""} - } - return &Blob{ - ID: id, - repo: repo, - }, nil -} diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index 552ae2bb8c..dff3a43fa3 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -5,10 +5,15 @@ package git import ( + "bufio" + "bytes" "context" "errors" "fmt" + "io" "strings" + + "code.gitea.io/gitea/modules/log" ) // BranchPrefix base dir of the branch information file store on git @@ -157,3 +162,180 @@ func (repo *Repository) RenameBranch(from, to string) error { _, _, err := NewCommand(repo.Ctx, "branch", "-m").AddDynamicArguments(from, to).RunStdString(&RunOpts{Dir: repo.Path}) return err } + +// IsObjectExist returns true if given reference exists in the repository. +func (repo *Repository) IsObjectExist(name string) bool { + if name == "" { + return false + } + + wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) + defer cancel() + _, err := wr.Write([]byte(name + "\n")) + if err != nil { + log.Debug("Error writing to CatFileBatchCheck %v", err) + return false + } + sha, _, _, err := ReadBatchLine(rd) + return err == nil && bytes.HasPrefix(sha, []byte(strings.TrimSpace(name))) +} + +// IsReferenceExist returns true if given reference exists in the repository. +func (repo *Repository) IsReferenceExist(name string) bool { + if name == "" { + return false + } + + wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) + defer cancel() + _, err := wr.Write([]byte(name + "\n")) + if err != nil { + log.Debug("Error writing to CatFileBatchCheck %v", err) + return false + } + _, _, _, err = ReadBatchLine(rd) + return err == nil +} + +// IsBranchExist returns true if given branch exists in current repository. +func (repo *Repository) IsBranchExist(name string) bool { + if repo == nil || name == "" { + return false + } + + return repo.IsReferenceExist(BranchPrefix + name) +} + +// GetBranchNames returns branches from the repository, skipping "skip" initial branches and +// returning at most "limit" branches, or all branches if "limit" is 0. +func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { + return callShowRef(repo.Ctx, repo.Path, BranchPrefix, TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"}, skip, limit) +} + +// WalkReferences walks all the references from the repository +// refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty. +func (repo *Repository) WalkReferences(refType ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { + var args TrustedCmdArgs + switch refType { + case ObjectTag: + args = TrustedCmdArgs{TagPrefix, "--sort=-taggerdate"} + case ObjectBranch: + args = TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"} + } + + return WalkShowRef(repo.Ctx, repo.Path, args, skip, limit, walkfn) +} + +// callShowRef return refs, if limit = 0 it will not limit +func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs TrustedCmdArgs, skip, limit int) (branchNames []string, countAll int, err error) { + countAll, err = WalkShowRef(ctx, repoPath, extraArgs, skip, limit, func(_, branchName string) error { + branchName = strings.TrimPrefix(branchName, trimPrefix) + branchNames = append(branchNames, branchName) + + return nil + }) + return branchNames, countAll, err +} + +func WalkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + args := TrustedCmdArgs{"for-each-ref", "--format=%(objectname) %(refname)"} + args = append(args, extraArgs...) + err := NewCommand(ctx, args...).Run(&RunOpts{ + Dir: repoPath, + Stdout: stdoutWriter, + Stderr: stderrBuilder, + }) + if err != nil { + if stderrBuilder.Len() == 0 { + _ = stdoutWriter.Close() + return + } + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + i := 0 + bufReader := bufio.NewReader(stdoutReader) + for i < skip { + _, isPrefix, err := bufReader.ReadLine() + if err == io.EOF { + return i, nil + } + if err != nil { + return 0, err + } + if !isPrefix { + i++ + } + } + for limit == 0 || i < skip+limit { + // The output of show-ref is simply a list: + // SP LF + sha, err := bufReader.ReadString(' ') + if err == io.EOF { + return i, nil + } + if err != nil { + return 0, err + } + + branchName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This shouldn't happen... but we'll tolerate it for the sake of peace + return i, nil + } + if err != nil { + return i, err + } + + if len(branchName) > 0 { + branchName = branchName[:len(branchName)-1] + } + + if len(sha) > 0 { + sha = sha[:len(sha)-1] + } + + err = walkfn(sha, branchName) + if err != nil { + return i, err + } + i++ + } + // count all refs + for limit != 0 { + _, isPrefix, err := bufReader.ReadLine() + if err == io.EOF { + return i, nil + } + if err != nil { + return 0, err + } + if !isPrefix { + i++ + } + } + return i, nil +} + +// GetRefsBySha returns all references filtered with prefix that belong to a sha commit hash +func (repo *Repository) GetRefsBySha(sha, prefix string) ([]string, error) { + var revList []string + _, err := WalkShowRef(repo.Ctx, repo.Path, nil, 0, 0, func(walkSha, refname string) error { + if walkSha == sha && strings.HasPrefix(refname, prefix) { + revList = append(revList, refname) + } + return nil + }) + return revList, err +} diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go deleted file mode 100644 index d1ec14d811..0000000000 --- a/modules/git/repo_branch_gogit.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "sort" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/storer" -) - -// IsObjectExist returns true if given reference exists in the repository. -func (repo *Repository) IsObjectExist(name string) bool { - if name == "" { - return false - } - - _, err := repo.gogitRepo.ResolveRevision(plumbing.Revision(name)) - - return err == nil -} - -// IsReferenceExist returns true if given reference exists in the repository. -func (repo *Repository) IsReferenceExist(name string) bool { - if name == "" { - return false - } - - reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) - if err != nil { - return false - } - return reference.Type() != plumbing.InvalidReference -} - -// IsBranchExist returns true if given branch exists in current repository. -func (repo *Repository) IsBranchExist(name string) bool { - if name == "" { - return false - } - reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true) - if err != nil { - return false - } - return reference.Type() != plumbing.InvalidReference -} - -// GetBranches returns branches from the repository, skipping "skip" initial branches and -// returning at most "limit" branches, or all branches if "limit" is 0. -// Branches are returned with sort of `-commiterdate` as the nogogit -// implementation. This requires full fetch, sort and then the -// skip/limit applies later as gogit returns in undefined order. -func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { - type BranchData struct { - name string - committerDate int64 - } - var branchData []BranchData - - branchIter, err := repo.gogitRepo.Branches() - if err != nil { - return nil, 0, err - } - - _ = branchIter.ForEach(func(branch *plumbing.Reference) error { - obj, err := repo.gogitRepo.CommitObject(branch.Hash()) - if err != nil { - // skip branch if can't find commit - return nil - } - - branchData = append(branchData, BranchData{strings.TrimPrefix(branch.Name().String(), BranchPrefix), obj.Committer.When.Unix()}) - return nil - }) - - sort.Slice(branchData, func(i, j int) bool { - return !(branchData[i].committerDate < branchData[j].committerDate) - }) - - var branchNames []string - maxPos := len(branchData) - if limit > 0 { - maxPos = min(skip+limit, maxPos) - } - for i := skip; i < maxPos; i++ { - branchNames = append(branchNames, branchData[i].name) - } - - return branchNames, len(branchData), nil -} - -// WalkReferences walks all the references from the repository -func (repo *Repository) WalkReferences(arg ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { - i := 0 - var iter storer.ReferenceIter - var err error - switch arg { - case ObjectTag: - iter, err = repo.gogitRepo.Tags() - case ObjectBranch: - iter, err = repo.gogitRepo.Branches() - default: - iter, err = repo.gogitRepo.References() - } - if err != nil { - return i, err - } - defer iter.Close() - - err = iter.ForEach(func(ref *plumbing.Reference) error { - if i < skip { - i++ - return nil - } - err := walkfn(ref.Hash().String(), string(ref.Name())) - i++ - if err != nil { - return err - } - if limit != 0 && i >= skip+limit { - return storer.ErrStop - } - return nil - }) - return i, err -} - -// GetRefsBySha returns all references filtered with prefix that belong to a sha commit hash -func (repo *Repository) GetRefsBySha(sha, prefix string) ([]string, error) { - var revList []string - iter, err := repo.gogitRepo.References() - if err != nil { - return nil, err - } - err = iter.ForEach(func(ref *plumbing.Reference) error { - if ref.Hash().String() == sha && strings.HasPrefix(string(ref.Name()), prefix) { - revList = append(revList, string(ref.Name())) - } - return nil - }) - return revList, err -} diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go deleted file mode 100644 index 470faebe25..0000000000 --- a/modules/git/repo_branch_nogogit.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "bufio" - "bytes" - "context" - "io" - "strings" - - "code.gitea.io/gitea/modules/log" -) - -// IsObjectExist returns true if given reference exists in the repository. -func (repo *Repository) IsObjectExist(name string) bool { - if name == "" { - return false - } - - wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) - defer cancel() - _, err := wr.Write([]byte(name + "\n")) - if err != nil { - log.Debug("Error writing to CatFileBatchCheck %v", err) - return false - } - sha, _, _, err := ReadBatchLine(rd) - return err == nil && bytes.HasPrefix(sha, []byte(strings.TrimSpace(name))) -} - -// IsReferenceExist returns true if given reference exists in the repository. -func (repo *Repository) IsReferenceExist(name string) bool { - if name == "" { - return false - } - - wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) - defer cancel() - _, err := wr.Write([]byte(name + "\n")) - if err != nil { - log.Debug("Error writing to CatFileBatchCheck %v", err) - return false - } - _, _, _, err = ReadBatchLine(rd) - return err == nil -} - -// IsBranchExist returns true if given branch exists in current repository. -func (repo *Repository) IsBranchExist(name string) bool { - if repo == nil || name == "" { - return false - } - - return repo.IsReferenceExist(BranchPrefix + name) -} - -// GetBranchNames returns branches from the repository, skipping "skip" initial branches and -// returning at most "limit" branches, or all branches if "limit" is 0. -func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { - return callShowRef(repo.Ctx, repo.Path, BranchPrefix, TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"}, skip, limit) -} - -// WalkReferences walks all the references from the repository -// refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty. -func (repo *Repository) WalkReferences(refType ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { - var args TrustedCmdArgs - switch refType { - case ObjectTag: - args = TrustedCmdArgs{TagPrefix, "--sort=-taggerdate"} - case ObjectBranch: - args = TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"} - } - - return WalkShowRef(repo.Ctx, repo.Path, args, skip, limit, walkfn) -} - -// callShowRef return refs, if limit = 0 it will not limit -func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs TrustedCmdArgs, skip, limit int) (branchNames []string, countAll int, err error) { - countAll, err = WalkShowRef(ctx, repoPath, extraArgs, skip, limit, func(_, branchName string) error { - branchName = strings.TrimPrefix(branchName, trimPrefix) - branchNames = append(branchNames, branchName) - - return nil - }) - return branchNames, countAll, err -} - -func WalkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) { - stdoutReader, stdoutWriter := io.Pipe() - defer func() { - _ = stdoutReader.Close() - _ = stdoutWriter.Close() - }() - - go func() { - stderrBuilder := &strings.Builder{} - args := TrustedCmdArgs{"for-each-ref", "--format=%(objectname) %(refname)"} - args = append(args, extraArgs...) - err := NewCommand(ctx, args...).Run(&RunOpts{ - Dir: repoPath, - Stdout: stdoutWriter, - Stderr: stderrBuilder, - }) - if err != nil { - if stderrBuilder.Len() == 0 { - _ = stdoutWriter.Close() - return - } - _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) - } else { - _ = stdoutWriter.Close() - } - }() - - i := 0 - bufReader := bufio.NewReader(stdoutReader) - for i < skip { - _, isPrefix, err := bufReader.ReadLine() - if err == io.EOF { - return i, nil - } - if err != nil { - return 0, err - } - if !isPrefix { - i++ - } - } - for limit == 0 || i < skip+limit { - // The output of show-ref is simply a list: - // SP LF - sha, err := bufReader.ReadString(' ') - if err == io.EOF { - return i, nil - } - if err != nil { - return 0, err - } - - branchName, err := bufReader.ReadString('\n') - if err == io.EOF { - // This shouldn't happen... but we'll tolerate it for the sake of peace - return i, nil - } - if err != nil { - return i, err - } - - if len(branchName) > 0 { - branchName = branchName[:len(branchName)-1] - } - - if len(sha) > 0 { - sha = sha[:len(sha)-1] - } - - err = walkfn(sha, branchName) - if err != nil { - return i, err - } - i++ - } - // count all refs - for limit != 0 { - _, isPrefix, err := bufReader.ReadLine() - if err == io.EOF { - return i, nil - } - if err != nil { - return 0, err - } - if !isPrefix { - i++ - } - } - return i, nil -} - -// GetRefsBySha returns all references filtered with prefix that belong to a sha commit hash -func (repo *Repository) GetRefsBySha(sha, prefix string) ([]string, error) { - var revList []string - _, err := WalkShowRef(repo.Ctx, repo.Path, nil, 0, 0, func(walkSha, refname string) error { - if walkSha == sha && strings.HasPrefix(refname, prefix) { - revList = append(revList, refname) - } - return nil - }) - return revList, err -} diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index f9168bef7e..2a325d3644 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -5,12 +5,15 @@ package git import ( + "bufio" "bytes" + "errors" "io" "strconv" "strings" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) @@ -513,3 +516,149 @@ func (repo *Repository) AddLastCommitCache(cacheKey, fullName, sha string) error } return nil } + +// ResolveReference resolves a name to a reference +func (repo *Repository) ResolveReference(name string) (string, error) { + stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--hash").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + if err != nil { + if strings.Contains(err.Error(), "not a valid ref") { + return "", ErrNotExist{name, ""} + } + return "", err + } + stdout = strings.TrimSpace(stdout) + if stdout == "" { + return "", ErrNotExist{name, ""} + } + + return stdout, nil +} + +// GetRefCommitID returns the last commit ID string of given reference (branch or tag). +func (repo *Repository) GetRefCommitID(name string) (string, error) { + wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) + defer cancel() + _, err := wr.Write([]byte(name + "\n")) + if err != nil { + return "", err + } + shaBs, _, _, err := ReadBatchLine(rd) + if IsErrNotExist(err) { + return "", ErrNotExist{name, ""} + } + + return string(shaBs), nil +} + +// SetReference sets the commit ID string of given reference (e.g. branch or tag). +func (repo *Repository) SetReference(name, commitID string) error { + _, _, err := NewCommand(repo.Ctx, "update-ref").AddDynamicArguments(name, commitID).RunStdString(&RunOpts{Dir: repo.Path}) + return err +} + +// RemoveReference removes the given reference (e.g. branch or tag). +func (repo *Repository) RemoveReference(name string) error { + _, _, err := NewCommand(repo.Ctx, "update-ref", "--no-deref", "-d").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + return err +} + +// IsCommitExist returns true if given commit exists in current repository. +func (repo *Repository) IsCommitExist(name string) bool { + _, _, err := NewCommand(repo.Ctx, "cat-file", "-e").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) + return err == nil +} + +func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { + wr, rd, cancel := repo.CatFileBatch(repo.Ctx) + defer cancel() + + _, _ = wr.Write([]byte(id.String() + "\n")) + + return repo.getCommitFromBatchReader(rd, id) +} + +func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) (*Commit, error) { + _, typ, size, err := ReadBatchLine(rd) + if err != nil { + if errors.Is(err, io.EOF) || IsErrNotExist(err) { + return nil, ErrNotExist{ID: id.String()} + } + return nil, err + } + + switch typ { + case "missing": + return nil, ErrNotExist{ID: id.String()} + case "tag": + // then we need to parse the tag + // and load the commit + data, err := io.ReadAll(io.LimitReader(rd, size)) + if err != nil { + return nil, err + } + _, err = rd.Discard(1) + if err != nil { + return nil, err + } + tag, err := parseTagData(id.Type(), data) + if err != nil { + return nil, err + } + + commit, err := tag.Commit(repo) + if err != nil { + return nil, err + } + + return commit, nil + case "commit": + commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size)) + if err != nil { + return nil, err + } + _, err = rd.Discard(1) + if err != nil { + return nil, err + } + + return commit, nil + default: + log.Debug("Unknown typ: %s", typ) + if err := DiscardFull(rd, size+1); err != nil { + return nil, err + } + return nil, ErrNotExist{ + ID: id.String(), + } + } +} + +// ConvertToGitID returns a GitHash object from a potential ID string +func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) { + objectFormat, err := repo.GetObjectFormat() + if err != nil { + return nil, err + } + if len(commitID) == objectFormat.FullLength() && objectFormat.IsValid(commitID) { + ID, err := NewIDFromString(commitID) + if err == nil { + return ID, nil + } + } + + wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) + defer cancel() + _, err = wr.Write([]byte(commitID + "\n")) + if err != nil { + return nil, err + } + sha, _, _, err := ReadBatchLine(rd) + if err != nil { + if IsErrNotExist(err) { + return nil, ErrNotExist{commitID, ""} + } + return nil, err + } + + return MustIDFromString(string(sha)), nil +} diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go deleted file mode 100644 index 84580be9a5..0000000000 --- a/modules/git/repo_commit_gogit.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/hash" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// GetRefCommitID returns the last commit ID string of given reference (branch or tag). -func (repo *Repository) GetRefCommitID(name string) (string, error) { - ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) - if err != nil { - if err == plumbing.ErrReferenceNotFound { - return "", ErrNotExist{ - ID: name, - } - } - return "", err - } - - return ref.Hash().String(), nil -} - -// SetReference sets the commit ID string of given reference (e.g. branch or tag). -func (repo *Repository) SetReference(name, commitID string) error { - return repo.gogitRepo.Storer.SetReference(plumbing.NewReferenceFromStrings(name, commitID)) -} - -// RemoveReference removes the given reference (e.g. branch or tag). -func (repo *Repository) RemoveReference(name string) error { - return repo.gogitRepo.Storer.RemoveReference(plumbing.ReferenceName(name)) -} - -// ConvertToHash returns a Hash object from a potential ID string -func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) { - objectFormat, err := repo.GetObjectFormat() - if err != nil { - return nil, err - } - if len(commitID) == hash.HexSize && objectFormat.IsValid(commitID) { - ID, err := NewIDFromString(commitID) - if err == nil { - return ID, nil - } - } - - actualCommitID, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(commitID).RunStdString(&RunOpts{Dir: repo.Path}) - actualCommitID = strings.TrimSpace(actualCommitID) - if err != nil { - if strings.Contains(err.Error(), "unknown revision or path") || - strings.Contains(err.Error(), "fatal: Needed a single revision") { - return objectFormat.EmptyObjectID(), ErrNotExist{commitID, ""} - } - return objectFormat.EmptyObjectID(), err - } - - return NewIDFromString(actualCommitID) -} - -// IsCommitExist returns true if given commit exists in current repository. -func (repo *Repository) IsCommitExist(name string) bool { - hash, err := repo.ConvertToGitID(name) - if err != nil { - return false - } - _, err = repo.gogitRepo.CommitObject(plumbing.Hash(hash.RawValue())) - return err == nil -} - -func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { - var tagObject *object.Tag - - commitID := plumbing.Hash(id.RawValue()) - gogitCommit, err := repo.gogitRepo.CommitObject(commitID) - if err == plumbing.ErrObjectNotFound { - tagObject, err = repo.gogitRepo.TagObject(commitID) - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - ID: id.String(), - } - } - if err == nil { - gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target) - } - // if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500 - } - if err != nil { - return nil, err - } - - commit := convertCommit(gogitCommit) - commit.repo = repo - - tree, err := gogitCommit.Tree() - if err != nil { - return nil, err - } - - commit.Tree.ID = ParseGogitHash(tree.Hash) - commit.Tree.gogitTree = tree - - return commit, nil -} diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go deleted file mode 100644 index ae4c21aaa3..0000000000 --- a/modules/git/repo_commit_nogogit.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "bufio" - "errors" - "io" - "strings" - - "code.gitea.io/gitea/modules/log" -) - -// ResolveReference resolves a name to a reference -func (repo *Repository) ResolveReference(name string) (string, error) { - stdout, _, err := NewCommand(repo.Ctx, "show-ref", "--hash").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) - if err != nil { - if strings.Contains(err.Error(), "not a valid ref") { - return "", ErrNotExist{name, ""} - } - return "", err - } - stdout = strings.TrimSpace(stdout) - if stdout == "" { - return "", ErrNotExist{name, ""} - } - - return stdout, nil -} - -// GetRefCommitID returns the last commit ID string of given reference (branch or tag). -func (repo *Repository) GetRefCommitID(name string) (string, error) { - wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) - defer cancel() - _, err := wr.Write([]byte(name + "\n")) - if err != nil { - return "", err - } - shaBs, _, _, err := ReadBatchLine(rd) - if IsErrNotExist(err) { - return "", ErrNotExist{name, ""} - } - - return string(shaBs), nil -} - -// SetReference sets the commit ID string of given reference (e.g. branch or tag). -func (repo *Repository) SetReference(name, commitID string) error { - _, _, err := NewCommand(repo.Ctx, "update-ref").AddDynamicArguments(name, commitID).RunStdString(&RunOpts{Dir: repo.Path}) - return err -} - -// RemoveReference removes the given reference (e.g. branch or tag). -func (repo *Repository) RemoveReference(name string) error { - _, _, err := NewCommand(repo.Ctx, "update-ref", "--no-deref", "-d").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) - return err -} - -// IsCommitExist returns true if given commit exists in current repository. -func (repo *Repository) IsCommitExist(name string) bool { - _, _, err := NewCommand(repo.Ctx, "cat-file", "-e").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repo.Path}) - return err == nil -} - -func (repo *Repository) getCommit(id ObjectID) (*Commit, error) { - wr, rd, cancel := repo.CatFileBatch(repo.Ctx) - defer cancel() - - _, _ = wr.Write([]byte(id.String() + "\n")) - - return repo.getCommitFromBatchReader(rd, id) -} - -func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id ObjectID) (*Commit, error) { - _, typ, size, err := ReadBatchLine(rd) - if err != nil { - if errors.Is(err, io.EOF) || IsErrNotExist(err) { - return nil, ErrNotExist{ID: id.String()} - } - return nil, err - } - - switch typ { - case "missing": - return nil, ErrNotExist{ID: id.String()} - case "tag": - // then we need to parse the tag - // and load the commit - data, err := io.ReadAll(io.LimitReader(rd, size)) - if err != nil { - return nil, err - } - _, err = rd.Discard(1) - if err != nil { - return nil, err - } - tag, err := parseTagData(id.Type(), data) - if err != nil { - return nil, err - } - - commit, err := tag.Commit(repo) - if err != nil { - return nil, err - } - - return commit, nil - case "commit": - commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size)) - if err != nil { - return nil, err - } - _, err = rd.Discard(1) - if err != nil { - return nil, err - } - - return commit, nil - default: - log.Debug("Unknown typ: %s", typ) - if err := DiscardFull(rd, size+1); err != nil { - return nil, err - } - return nil, ErrNotExist{ - ID: id.String(), - } - } -} - -// ConvertToGitID returns a GitHash object from a potential ID string -func (repo *Repository) ConvertToGitID(commitID string) (ObjectID, error) { - objectFormat, err := repo.GetObjectFormat() - if err != nil { - return nil, err - } - if len(commitID) == objectFormat.FullLength() && objectFormat.IsValid(commitID) { - ID, err := NewIDFromString(commitID) - if err == nil { - return ID, nil - } - } - - wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) - defer cancel() - _, err = wr.Write([]byte(commitID + "\n")) - if err != nil { - return nil, err - } - sha, _, _, err := ReadBatchLine(rd) - if err != nil { - if IsErrNotExist(err) { - return nil, ErrNotExist{commitID, ""} - } - return nil, err - } - - return MustIDFromString(string(sha)), nil -} diff --git a/modules/git/repo_commitgraph_gogit.go b/modules/git/repo_commitgraph_gogit.go deleted file mode 100644 index d3182f15c6..0000000000 --- a/modules/git/repo_commitgraph_gogit.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 The Gitea Authors. -// All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "os" - "path" - - gitealog "code.gitea.io/gitea/modules/log" - - commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2" - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" -) - -// CommitNodeIndex returns the index for walking commit graph -func (r *Repository) CommitNodeIndex() (cgobject.CommitNodeIndex, *os.File) { - indexPath := path.Join(r.Path, "objects", "info", "commit-graph") - - file, err := os.Open(indexPath) - if err == nil { - var index commitgraph.Index - index, err = commitgraph.OpenFileIndex(file) - if err == nil { - return cgobject.NewGraphCommitNodeIndex(index, r.gogitRepo.Storer), file - } - } - - if !os.IsNotExist(err) { - gitealog.Warn("Unable to read commit-graph for %s: %v", r.Path, err) - } - - return cgobject.NewObjectCommitNodeIndex(r.gogitRepo.Storer), nil -} diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go index c40d6937b5..84638b3cef 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/repo_language_stats.go @@ -4,8 +4,17 @@ package git import ( + "bytes" + "cmp" + "io" "strings" "unicode" + + "code.gitea.io/gitea/modules/analyze" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/optional" + + "github.com/go-enry/go-enry/v2" ) const ( @@ -46,3 +55,194 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 { } return res } + +// GetLanguageStats calculates language stats for git repository at specified commit +func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { + // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. + // so let's create a batch stdin and stdout + batchStdinWriter, batchReader, cancel := repo.CatFileBatch(repo.Ctx) + defer cancel() + + writeID := func(id string) error { + _, err := batchStdinWriter.Write([]byte(id + "\n")) + return err + } + + if err := writeID(commitID); err != nil { + return nil, err + } + shaBytes, typ, size, err := ReadBatchLine(batchReader) + if typ != "commit" { + log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) + return nil, ErrNotExist{commitID, ""} + } + + sha, err := NewIDFromString(string(shaBytes)) + if err != nil { + log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) + return nil, ErrNotExist{commitID, ""} + } + + commit, err := CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) + if err != nil { + log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) + return nil, err + } + if _, err = batchReader.Discard(1); err != nil { + return nil, err + } + + tree := commit.Tree + + entries, err := tree.ListEntriesRecursiveWithSize() + if err != nil { + return nil, err + } + + checker, err := repo.GitAttributeChecker(commitID, LinguistAttributes...) + if err != nil { + return nil, err + } + defer checker.Close() + + contentBuf := bytes.Buffer{} + var content []byte + + // sizes contains the current calculated size of all files by language + sizes := make(map[string]int64) + // by default we will only count the sizes of programming languages or markup languages + // unless they are explicitly set using linguist-language + includedLanguage := map[string]bool{} + // or if there's only one language in the repository + firstExcludedLanguage := "" + firstExcludedLanguageSize := int64(0) + + isTrue := func(v optional.Option[bool]) bool { + return v.ValueOrDefault(false) + } + isFalse := func(v optional.Option[bool]) bool { + return !v.ValueOrDefault(true) + } + + for _, f := range entries { + select { + case <-repo.Ctx.Done(): + return sizes, repo.Ctx.Err() + default: + } + + contentBuf.Reset() + content = contentBuf.Bytes() + + if f.Size() == 0 { + continue + } + + isVendored := optional.None[bool]() + isGenerated := optional.None[bool]() + isDocumentation := optional.None[bool]() + isDetectable := optional.None[bool]() + + attrs, err := checker.CheckPath(f.Name()) + if err == nil { + isVendored = attrs["linguist-vendored"].Bool() + isGenerated = attrs["linguist-generated"].Bool() + isDocumentation = attrs["linguist-documentation"].Bool() + isDetectable = attrs["linguist-detectable"].Bool() + if language := cmp.Or( + attrs["linguist-language"].String(), + attrs["gitlab-language"].Prefix(), + ); language != "" { + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if len(group) != 0 { + language = group + } + + // this language will always be added to the size + sizes[language] += f.Size() + continue + } + } + + if isFalse(isDetectable) || isTrue(isVendored) || isTrue(isDocumentation) || + (!isFalse(isVendored) && analyze.IsVendor(f.Name())) || + enry.IsDotFile(f.Name()) || + enry.IsConfiguration(f.Name()) || + (!isFalse(isDocumentation) && enry.IsDocumentation(f.Name())) { + continue + } + + // If content can not be read or file is too big just do detection by filename + + if f.Size() <= bigFileSize { + if err := writeID(f.ID.String()); err != nil { + return nil, err + } + _, _, size, err := ReadBatchLine(batchReader) + if err != nil { + log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) + return nil, err + } + + sizeToRead := size + discard := int64(1) + if size > fileSizeLimit { + sizeToRead = fileSizeLimit + discard = size - fileSizeLimit + 1 + } + + _, err = contentBuf.ReadFrom(io.LimitReader(batchReader, sizeToRead)) + if err != nil { + return nil, err + } + content = contentBuf.Bytes() + if err := DiscardFull(batchReader, discard); err != nil { + return nil, err + } + } + if !isTrue(isGenerated) && enry.IsGenerated(f.Name(), content) { + continue + } + + // FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary? + // - eg. do the all the detection tests using filename first before reading content. + language := analyze.GetCodeLanguage(f.Name(), content) + if language == "" { + continue + } + + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if group != "" { + language = group + } + + included, checked := includedLanguage[language] + langType := enry.GetLanguageType(language) + if !checked { + included = langType == enry.Programming || langType == enry.Markup + if !included && (isTrue(isDetectable) || (langType == enry.Prose && isFalse(isDocumentation))) { + included = true + } + includedLanguage[language] = included + } + if included { + sizes[language] += f.Size() + } else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) { + // Only consider Programming or Markup languages as fallback + if !(langType == enry.Programming || langType == enry.Markup) { + continue + } + firstExcludedLanguage = language + firstExcludedLanguageSize += f.Size() + } + } + + // If there are no included languages add the first excluded language + if len(sizes) == 0 && firstExcludedLanguage != "" { + sizes[firstExcludedLanguage] = firstExcludedLanguageSize + } + + return mergeLanguageStats(sizes), nil +} diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/repo_language_stats_gogit.go deleted file mode 100644 index 1276ce1a44..0000000000 --- a/modules/git/repo_language_stats_gogit.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// Copyright 2024 The Forgejo Authors c/o Codeberg e.V.. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "bytes" - "io" - "strings" - - "code.gitea.io/gitea/modules/analyze" - "code.gitea.io/gitea/modules/optional" - - "github.com/go-enry/go-enry/v2" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { - r, err := git.PlainOpen(repo.Path) - if err != nil { - return nil, err - } - - rev, err := r.ResolveRevision(plumbing.Revision(commitID)) - if err != nil { - return nil, err - } - - commit, err := r.CommitObject(*rev) - if err != nil { - return nil, err - } - - tree, err := commit.Tree() - if err != nil { - return nil, err - } - - checker, deferable := repo.CheckAttributeReader(commitID) - defer deferable() - - // sizes contains the current calculated size of all files by language - sizes := make(map[string]int64) - // by default we will only count the sizes of programming languages or markup languages - // unless they are explicitly set using linguist-language - includedLanguage := map[string]bool{} - // or if there's only one language in the repository - firstExcludedLanguage := "" - firstExcludedLanguageSize := int64(0) - - isTrue := func(v optional.Option[bool]) bool { - return v.ValueOrDefault(false) - } - isFalse := func(v optional.Option[bool]) bool { - return !v.ValueOrDefault(true) - } - - err = tree.Files().ForEach(func(f *object.File) error { - if f.Size == 0 { - return nil - } - - isVendored := optional.None[bool]() - isGenerated := optional.None[bool]() - isDocumentation := optional.None[bool]() - isDetectable := optional.None[bool]() - - if checker != nil { - attrs, err := checker.CheckPath(f.Name) - if err == nil { - isVendored = attributeToBool(attrs, "linguist-vendored") - isGenerated = attributeToBool(attrs, "linguist-generated") - isDocumentation = attributeToBool(attrs, "linguist-documentation") - isDetectable = attributeToBool(attrs, "linguist-detectable") - if language, has := attrs["linguist-language"]; has && language != "unspecified" && language != "" { - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } - - // this language will always be added to the size - sizes[language] += f.Size - return nil - } else if language, has := attrs["gitlab-language"]; has && language != "unspecified" && language != "" { - // strip off a ? if present - if idx := strings.IndexByte(language, '?'); idx >= 0 { - language = language[:idx] - } - if len(language) != 0 { - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } - - // this language will always be added to the size - sizes[language] += f.Size - return nil - } - } - } - } - - if isFalse(isDetectable) || isTrue(isVendored) || isTrue(isDocumentation) || - (!isFalse(isVendored) && analyze.IsVendor(f.Name)) || - enry.IsDotFile(f.Name) || - enry.IsConfiguration(f.Name) || - (!isFalse(isDocumentation) && enry.IsDocumentation(f.Name)) { - return nil - } - - // If content can not be read or file is too big just do detection by filename - var content []byte - if f.Size <= bigFileSize { - content, _ = readFile(f, fileSizeLimit) - } - if !isTrue(isGenerated) && enry.IsGenerated(f.Name, content) { - return nil - } - - // TODO: Use .gitattributes file for linguist overrides - language := analyze.GetCodeLanguage(f.Name, content) - if language == enry.OtherLanguage || language == "" { - return nil - } - - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if group != "" { - language = group - } - - included, checked := includedLanguage[language] - langType := enry.GetLanguageType(language) - if !checked { - included = langType == enry.Programming || langType == enry.Markup - if !included && (isTrue(isDetectable) || (langType == enry.Prose && isFalse(isDocumentation))) { - included = true - } - includedLanguage[language] = included - } - if included { - sizes[language] += f.Size - } else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) { - // Only consider Programming or Markup languages as fallback - if !(langType == enry.Programming || langType == enry.Markup) { - return nil - } - - firstExcludedLanguage = language - firstExcludedLanguageSize += f.Size - } - - return nil - }) - if err != nil { - return nil, err - } - - // If there are no included languages add the first excluded language - if len(sizes) == 0 && firstExcludedLanguage != "" { - sizes[firstExcludedLanguage] = firstExcludedLanguageSize - } - - return mergeLanguageStats(sizes), nil -} - -func readFile(f *object.File, limit int64) ([]byte, error) { - r, err := f.Reader() - if err != nil { - return nil, err - } - defer r.Close() - - if limit <= 0 { - return io.ReadAll(r) - } - - size := f.Size - if limit > 0 && size > limit { - size = limit - } - buf := bytes.NewBuffer(nil) - buf.Grow(int(size)) - _, err = io.Copy(buf, io.LimitReader(r, limit)) - return buf.Bytes(), err -} diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go deleted file mode 100644 index 672f7571d9..0000000000 --- a/modules/git/repo_language_stats_nogogit.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// Copyright 2024 The Forgejo Authors c/o Codeberg e.V.. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "bytes" - "cmp" - "io" - - "code.gitea.io/gitea/modules/analyze" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/optional" - - "github.com/go-enry/go-enry/v2" -) - -// GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { - // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. - // so let's create a batch stdin and stdout - batchStdinWriter, batchReader, cancel := repo.CatFileBatch(repo.Ctx) - defer cancel() - - writeID := func(id string) error { - _, err := batchStdinWriter.Write([]byte(id + "\n")) - return err - } - - if err := writeID(commitID); err != nil { - return nil, err - } - shaBytes, typ, size, err := ReadBatchLine(batchReader) - if typ != "commit" { - log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} - } - - sha, err := NewIDFromString(string(shaBytes)) - if err != nil { - log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, ErrNotExist{commitID, ""} - } - - commit, err := CommitFromReader(repo, sha, io.LimitReader(batchReader, size)) - if err != nil { - log.Debug("Unable to get commit for: %s. Err: %v", commitID, err) - return nil, err - } - if _, err = batchReader.Discard(1); err != nil { - return nil, err - } - - tree := commit.Tree - - entries, err := tree.ListEntriesRecursiveWithSize() - if err != nil { - return nil, err - } - - checker, err := repo.GitAttributeChecker(commitID, LinguistAttributes...) - if err != nil { - return nil, err - } - defer checker.Close() - - contentBuf := bytes.Buffer{} - var content []byte - - // sizes contains the current calculated size of all files by language - sizes := make(map[string]int64) - // by default we will only count the sizes of programming languages or markup languages - // unless they are explicitly set using linguist-language - includedLanguage := map[string]bool{} - // or if there's only one language in the repository - firstExcludedLanguage := "" - firstExcludedLanguageSize := int64(0) - - isTrue := func(v optional.Option[bool]) bool { - return v.ValueOrDefault(false) - } - isFalse := func(v optional.Option[bool]) bool { - return !v.ValueOrDefault(true) - } - - for _, f := range entries { - select { - case <-repo.Ctx.Done(): - return sizes, repo.Ctx.Err() - default: - } - - contentBuf.Reset() - content = contentBuf.Bytes() - - if f.Size() == 0 { - continue - } - - isVendored := optional.None[bool]() - isGenerated := optional.None[bool]() - isDocumentation := optional.None[bool]() - isDetectable := optional.None[bool]() - - attrs, err := checker.CheckPath(f.Name()) - if err == nil { - isVendored = attrs["linguist-vendored"].Bool() - isGenerated = attrs["linguist-generated"].Bool() - isDocumentation = attrs["linguist-documentation"].Bool() - isDetectable = attrs["linguist-detectable"].Bool() - if language := cmp.Or( - attrs["linguist-language"].String(), - attrs["gitlab-language"].Prefix(), - ); language != "" { - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if len(group) != 0 { - language = group - } - - // this language will always be added to the size - sizes[language] += f.Size() - continue - } - } - - if isFalse(isDetectable) || isTrue(isVendored) || isTrue(isDocumentation) || - (!isFalse(isVendored) && analyze.IsVendor(f.Name())) || - enry.IsDotFile(f.Name()) || - enry.IsConfiguration(f.Name()) || - (!isFalse(isDocumentation) && enry.IsDocumentation(f.Name())) { - continue - } - - // If content can not be read or file is too big just do detection by filename - - if f.Size() <= bigFileSize { - if err := writeID(f.ID.String()); err != nil { - return nil, err - } - _, _, size, err := ReadBatchLine(batchReader) - if err != nil { - log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) - return nil, err - } - - sizeToRead := size - discard := int64(1) - if size > fileSizeLimit { - sizeToRead = fileSizeLimit - discard = size - fileSizeLimit + 1 - } - - _, err = contentBuf.ReadFrom(io.LimitReader(batchReader, sizeToRead)) - if err != nil { - return nil, err - } - content = contentBuf.Bytes() - if err := DiscardFull(batchReader, discard); err != nil { - return nil, err - } - } - if !isTrue(isGenerated) && enry.IsGenerated(f.Name(), content) { - continue - } - - // FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary? - // - eg. do the all the detection tests using filename first before reading content. - language := analyze.GetCodeLanguage(f.Name(), content) - if language == "" { - continue - } - - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if group != "" { - language = group - } - - included, checked := includedLanguage[language] - langType := enry.GetLanguageType(language) - if !checked { - included = langType == enry.Programming || langType == enry.Markup - if !included && (isTrue(isDetectable) || (langType == enry.Prose && isFalse(isDocumentation))) { - included = true - } - includedLanguage[language] = included - } - if included { - sizes[language] += f.Size() - } else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) { - // Only consider Programming or Markup languages as fallback - if !(langType == enry.Programming || langType == enry.Markup) { - continue - } - firstExcludedLanguage = language - firstExcludedLanguageSize += f.Size() - } - } - - // If there are no included languages add the first excluded language - if len(sizes) == 0 && firstExcludedLanguage != "" { - sizes[firstExcludedLanguage] = firstExcludedLanguageSize - } - - return mergeLanguageStats(sizes), nil -} diff --git a/modules/git/repo_language_stats_test.go b/modules/git/repo_language_stats_test.go index 1ee5f4c3af..fd80e44a86 100644 --- a/modules/git/repo_language_stats_test.go +++ b/modules/git/repo_language_stats_test.go @@ -1,8 +1,6 @@ // Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build !gogit - package git import ( diff --git a/modules/git/repo_ref.go b/modules/git/repo_ref.go index b0c602c6a5..550c653729 100644 --- a/modules/git/repo_ref.go +++ b/modules/git/repo_ref.go @@ -4,8 +4,10 @@ package git import ( + "bufio" "context" "fmt" + "io" "strings" "code.gitea.io/gitea/modules/util" @@ -78,3 +80,78 @@ func (repo *Repository) ExpandRef(ref string) (string, error) { } return "", fmt.Errorf("could not expand reference '%s'", ref) } + +// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. +func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand(repo.Ctx, "for-each-ref").Run(&RunOpts{ + Dir: repo.Path, + Stdout: stdoutWriter, + Stderr: stderrBuilder, + }) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + refs := make([]*Reference, 0) + bufReader := bufio.NewReader(stdoutReader) + for { + // The output of for-each-ref is simply a list: + // SP TAB LF + sha, err := bufReader.ReadString(' ') + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + sha = sha[:len(sha)-1] + + typ, err := bufReader.ReadString('\t') + if err == io.EOF { + // This should not happen, but we'll tolerate it + break + } + if err != nil { + return nil, err + } + typ = typ[:len(typ)-1] + + refName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This should not happen, but we'll tolerate it + break + } + if err != nil { + return nil, err + } + refName = refName[:len(refName)-1] + + // refName cannot be HEAD but can be remotes or stash + if strings.HasPrefix(refName, RemotePrefix) || refName == "/refs/stash" { + continue + } + + if pattern == "" || strings.HasPrefix(refName, pattern) { + r := &Reference{ + Name: refName, + Object: MustIDFromString(sha), + Type: typ, + repo: repo, + } + refs = append(refs, r) + } + } + + return refs, nil +} diff --git a/modules/git/repo_ref_gogit.go b/modules/git/repo_ref_gogit.go deleted file mode 100644 index fc43ce5545..0000000000 --- a/modules/git/repo_ref_gogit.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "strings" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" -) - -// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. -func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { - r, err := git.PlainOpen(repo.Path) - if err != nil { - return nil, err - } - - refsIter, err := r.References() - if err != nil { - return nil, err - } - refs := make([]*Reference, 0) - if err = refsIter.ForEach(func(ref *plumbing.Reference) error { - if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() && - (pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) { - refType := string(ObjectCommit) - if ref.Name().IsTag() { - // tags can be of type `commit` (lightweight) or `tag` (annotated) - if tagType, _ := repo.GetTagType(ParseGogitHash(ref.Hash())); err == nil { - refType = tagType - } - } - r := &Reference{ - Name: ref.Name().String(), - Object: ParseGogitHash(ref.Hash()), - Type: refType, - repo: repo, - } - refs = append(refs, r) - } - return nil - }); err != nil { - return nil, err - } - - return refs, nil -} diff --git a/modules/git/repo_ref_nogogit.go b/modules/git/repo_ref_nogogit.go deleted file mode 100644 index ac53d661b5..0000000000 --- a/modules/git/repo_ref_nogogit.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "bufio" - "io" - "strings" -) - -// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. -func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { - stdoutReader, stdoutWriter := io.Pipe() - defer func() { - _ = stdoutReader.Close() - _ = stdoutWriter.Close() - }() - - go func() { - stderrBuilder := &strings.Builder{} - err := NewCommand(repo.Ctx, "for-each-ref").Run(&RunOpts{ - Dir: repo.Path, - Stdout: stdoutWriter, - Stderr: stderrBuilder, - }) - if err != nil { - _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) - } else { - _ = stdoutWriter.Close() - } - }() - - refs := make([]*Reference, 0) - bufReader := bufio.NewReader(stdoutReader) - for { - // The output of for-each-ref is simply a list: - // SP TAB LF - sha, err := bufReader.ReadString(' ') - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - sha = sha[:len(sha)-1] - - typ, err := bufReader.ReadString('\t') - if err == io.EOF { - // This should not happen, but we'll tolerate it - break - } - if err != nil { - return nil, err - } - typ = typ[:len(typ)-1] - - refName, err := bufReader.ReadString('\n') - if err == io.EOF { - // This should not happen, but we'll tolerate it - break - } - if err != nil { - return nil, err - } - refName = refName[:len(refName)-1] - - // refName cannot be HEAD but can be remotes or stash - if strings.HasPrefix(refName, RemotePrefix) || refName == "/refs/stash" { - continue - } - - if pattern == "" || strings.HasPrefix(refName, pattern) { - r := &Reference{ - Name: refName, - Object: MustIDFromString(sha), - Type: typ, - repo: repo, - } - refs = append(refs, r) - } - } - - return refs, nil -} diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 638c508e4b..d925d4a7d3 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -6,11 +6,13 @@ package git import ( "context" + "errors" "fmt" "io" "strings" "code.gitea.io/gitea/modules/git/foreachref" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" ) @@ -236,3 +238,123 @@ func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) { } return tag, nil } + +// IsTagExist returns true if given tag exists in the repository. +func (repo *Repository) IsTagExist(name string) bool { + if repo == nil || name == "" { + return false + } + + return repo.IsReferenceExist(TagPrefix + name) +} + +// GetTags returns all tags of the repository. +// returning at most limit tags, or all if limit is 0. +func (repo *Repository) GetTags(skip, limit int) (tags []string, err error) { + tags, _, err = callShowRef(repo.Ctx, repo.Path, TagPrefix, TrustedCmdArgs{TagPrefix, "--sort=-taggerdate"}, skip, limit) + return tags, err +} + +// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) +func (repo *Repository) GetTagType(id ObjectID) (string, error) { + wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) + defer cancel() + _, err := wr.Write([]byte(id.String() + "\n")) + if err != nil { + return "", err + } + _, typ, _, err := ReadBatchLine(rd) + if IsErrNotExist(err) { + return "", ErrNotExist{ID: id.String()} + } + return typ, nil +} + +func (repo *Repository) getTag(tagID ObjectID, name string) (*Tag, error) { + t, ok := repo.tagCache.Get(tagID.String()) + if ok { + log.Debug("Hit cache: %s", tagID) + tagClone := *t.(*Tag) + tagClone.Name = name // This is necessary because lightweight tags may have same id + return &tagClone, nil + } + + tp, err := repo.GetTagType(tagID) + if err != nil { + return nil, err + } + + // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object + commitIDStr, err := repo.GetTagCommitID(name) + if err != nil { + // every tag should have a commit ID so return all errors + return nil, err + } + commitID, err := NewIDFromString(commitIDStr) + if err != nil { + return nil, err + } + + // If type is "commit, the tag is a lightweight tag + if ObjectType(tp) == ObjectCommit { + commit, err := repo.GetCommit(commitIDStr) + if err != nil { + return nil, err + } + tag := &Tag{ + Name: name, + ID: tagID, + Object: commitID, + Type: tp, + Tagger: commit.Committer, + Message: commit.Message(), + } + + repo.tagCache.Set(tagID.String(), tag) + return tag, nil + } + + // The tag is an annotated tag with a message. + wr, rd, cancel := repo.CatFileBatch(repo.Ctx) + defer cancel() + + if _, err := wr.Write([]byte(tagID.String() + "\n")); err != nil { + return nil, err + } + _, typ, size, err := ReadBatchLine(rd) + if err != nil { + if errors.Is(err, io.EOF) || IsErrNotExist(err) { + return nil, ErrNotExist{ID: tagID.String()} + } + return nil, err + } + if typ != "tag" { + if err := DiscardFull(rd, size+1); err != nil { + return nil, err + } + return nil, ErrNotExist{ID: tagID.String()} + } + + // then we need to parse the tag + // and load the commit + data, err := io.ReadAll(io.LimitReader(rd, size)) + if err != nil { + return nil, err + } + _, err = rd.Discard(1) + if err != nil { + return nil, err + } + + tag, err := parseTagData(tagID.Type(), data) + if err != nil { + return nil, err + } + + tag.Name = name + tag.ID = tagID + tag.Type = tp + + repo.tagCache.Set(tagID.String(), tag) + return tag, nil +} diff --git a/modules/git/repo_tag_gogit.go b/modules/git/repo_tag_gogit.go deleted file mode 100644 index 4a7a06e9bd..0000000000 --- a/modules/git/repo_tag_gogit.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "strings" - - "code.gitea.io/gitea/modules/log" - - "github.com/go-git/go-git/v5/plumbing" -) - -// IsTagExist returns true if given tag exists in the repository. -func (repo *Repository) IsTagExist(name string) bool { - _, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true) - return err == nil -} - -// GetTags returns all tags of the repository. -// returning at most limit tags, or all if limit is 0. -func (repo *Repository) GetTags(skip, limit int) ([]string, error) { - var tagNames []string - - tags, err := repo.gogitRepo.Tags() - if err != nil { - return nil, err - } - - _ = tags.ForEach(func(tag *plumbing.Reference) error { - tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) - return nil - }) - - // Reverse order - for i := 0; i < len(tagNames)/2; i++ { - j := len(tagNames) - i - 1 - tagNames[i], tagNames[j] = tagNames[j], tagNames[i] - } - - // since we have to reverse order we can paginate only afterwards - if len(tagNames) < skip { - tagNames = []string{} - } else { - tagNames = tagNames[skip:] - } - if limit != 0 && len(tagNames) > limit { - tagNames = tagNames[:limit] - } - - return tagNames, nil -} - -// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) -func (repo *Repository) GetTagType(id ObjectID) (string, error) { - // Get tag type - obj, err := repo.gogitRepo.Object(plumbing.AnyObject, plumbing.Hash(id.RawValue())) - if err != nil { - if err == plumbing.ErrReferenceNotFound { - return "", &ErrNotExist{ID: id.String()} - } - return "", err - } - - return obj.Type().String(), nil -} - -func (repo *Repository) getTag(tagID ObjectID, name string) (*Tag, error) { - t, ok := repo.tagCache.Get(tagID.String()) - if ok { - log.Debug("Hit cache: %s", tagID) - tagClone := *t.(*Tag) - tagClone.Name = name // This is necessary because lightweight tags may have same id - return &tagClone, nil - } - - tp, err := repo.GetTagType(tagID) - if err != nil { - return nil, err - } - - // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object - commitIDStr, err := repo.GetTagCommitID(name) - if err != nil { - // every tag should have a commit ID so return all errors - return nil, err - } - commitID, err := NewIDFromString(commitIDStr) - if err != nil { - return nil, err - } - - // If type is "commit, the tag is a lightweight tag - if ObjectType(tp) == ObjectCommit { - commit, err := repo.GetCommit(commitIDStr) - if err != nil { - return nil, err - } - tag := &Tag{ - Name: name, - ID: tagID, - Object: commitID, - Type: tp, - Tagger: commit.Committer, - Message: commit.Message(), - } - - repo.tagCache.Set(tagID.String(), tag) - return tag, nil - } - - gogitTag, err := repo.gogitRepo.TagObject(plumbing.Hash(tagID.RawValue())) - if err != nil { - if err == plumbing.ErrReferenceNotFound { - return nil, &ErrNotExist{ID: tagID.String()} - } - - return nil, err - } - - tag := &Tag{ - Name: name, - ID: tagID, - Object: commitID.Type().MustID(gogitTag.Target[:]), - Type: tp, - Tagger: &gogitTag.Tagger, - Message: gogitTag.Message, - } - - repo.tagCache.Set(tagID.String(), tag) - return tag, nil -} diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go deleted file mode 100644 index cbab39f8c5..0000000000 --- a/modules/git/repo_tag_nogogit.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "errors" - "io" - - "code.gitea.io/gitea/modules/log" -) - -// IsTagExist returns true if given tag exists in the repository. -func (repo *Repository) IsTagExist(name string) bool { - if repo == nil || name == "" { - return false - } - - return repo.IsReferenceExist(TagPrefix + name) -} - -// GetTags returns all tags of the repository. -// returning at most limit tags, or all if limit is 0. -func (repo *Repository) GetTags(skip, limit int) (tags []string, err error) { - tags, _, err = callShowRef(repo.Ctx, repo.Path, TagPrefix, TrustedCmdArgs{TagPrefix, "--sort=-taggerdate"}, skip, limit) - return tags, err -} - -// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) -func (repo *Repository) GetTagType(id ObjectID) (string, error) { - wr, rd, cancel := repo.CatFileBatchCheck(repo.Ctx) - defer cancel() - _, err := wr.Write([]byte(id.String() + "\n")) - if err != nil { - return "", err - } - _, typ, _, err := ReadBatchLine(rd) - if IsErrNotExist(err) { - return "", ErrNotExist{ID: id.String()} - } - return typ, nil -} - -func (repo *Repository) getTag(tagID ObjectID, name string) (*Tag, error) { - t, ok := repo.tagCache.Get(tagID.String()) - if ok { - log.Debug("Hit cache: %s", tagID) - tagClone := *t.(*Tag) - tagClone.Name = name // This is necessary because lightweight tags may have same id - return &tagClone, nil - } - - tp, err := repo.GetTagType(tagID) - if err != nil { - return nil, err - } - - // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object - commitIDStr, err := repo.GetTagCommitID(name) - if err != nil { - // every tag should have a commit ID so return all errors - return nil, err - } - commitID, err := NewIDFromString(commitIDStr) - if err != nil { - return nil, err - } - - // If type is "commit, the tag is a lightweight tag - if ObjectType(tp) == ObjectCommit { - commit, err := repo.GetCommit(commitIDStr) - if err != nil { - return nil, err - } - tag := &Tag{ - Name: name, - ID: tagID, - Object: commitID, - Type: tp, - Tagger: commit.Committer, - Message: commit.Message(), - } - - repo.tagCache.Set(tagID.String(), tag) - return tag, nil - } - - // The tag is an annotated tag with a message. - wr, rd, cancel := repo.CatFileBatch(repo.Ctx) - defer cancel() - - if _, err := wr.Write([]byte(tagID.String() + "\n")); err != nil { - return nil, err - } - _, typ, size, err := ReadBatchLine(rd) - if err != nil { - if errors.Is(err, io.EOF) || IsErrNotExist(err) { - return nil, ErrNotExist{ID: tagID.String()} - } - return nil, err - } - if typ != "tag" { - if err := DiscardFull(rd, size+1); err != nil { - return nil, err - } - return nil, ErrNotExist{ID: tagID.String()} - } - - // then we need to parse the tag - // and load the commit - data, err := io.ReadAll(io.LimitReader(rd, size)) - if err != nil { - return nil, err - } - _, err = rd.Discard(1) - if err != nil { - return nil, err - } - - tag, err := parseTagData(tagID.Type(), data) - if err != nil { - return nil, err - } - - tag.Name = name - tag.ID = tagID - tag.Type = tp - - repo.tagCache.Set(tagID.String(), tag) - return tag, nil -} diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index ab48d47d13..79a7e50eb4 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -6,6 +6,7 @@ package git import ( "bytes" + "io" "os" "strings" "time" @@ -65,3 +66,88 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt } return NewIDFromString(strings.TrimSpace(stdout.String())) } + +func (repo *Repository) getTree(id ObjectID) (*Tree, error) { + wr, rd, cancel := repo.CatFileBatch(repo.Ctx) + defer cancel() + + _, _ = wr.Write([]byte(id.String() + "\n")) + + // ignore the SHA + _, typ, size, err := ReadBatchLine(rd) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + resolvedID := id + data, err := io.ReadAll(io.LimitReader(rd, size)) + if err != nil { + return nil, err + } + tag, err := parseTagData(id.Type(), data) + if err != nil { + return nil, err + } + commit, err := tag.Commit(repo) + if err != nil { + return nil, err + } + commit.Tree.ResolvedID = resolvedID + return &commit.Tree, nil + case "commit": + commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size)) + if err != nil { + return nil, err + } + if _, err := rd.Discard(1); err != nil { + return nil, err + } + commit.Tree.ResolvedID = commit.ID + return &commit.Tree, nil + case "tree": + tree := NewTree(repo, id) + tree.ResolvedID = id + objectFormat, err := repo.GetObjectFormat() + if err != nil { + return nil, err + } + tree.entries, err = catBatchParseTreeEntries(objectFormat, tree, rd, size) + if err != nil { + return nil, err + } + tree.entriesParsed = true + return tree, nil + default: + if err := DiscardFull(rd, size+1); err != nil { + return nil, err + } + return nil, ErrNotExist{ + ID: id.String(), + } + } +} + +// GetTree find the tree object in the repository. +func (repo *Repository) GetTree(idStr string) (*Tree, error) { + objectFormat, err := repo.GetObjectFormat() + if err != nil { + return nil, err + } + if len(idStr) != objectFormat.FullLength() { + res, err := repo.GetRefCommitID(idStr) + if err != nil { + return nil, err + } + if len(res) > 0 { + idStr = res + } + } + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + + return repo.getTree(id) +} diff --git a/modules/git/repo_tree_gogit.go b/modules/git/repo_tree_gogit.go deleted file mode 100644 index dc97ce1344..0000000000 --- a/modules/git/repo_tree_gogit.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import "github.com/go-git/go-git/v5/plumbing" - -func (repo *Repository) getTree(id ObjectID) (*Tree, error) { - gogitTree, err := repo.gogitRepo.TreeObject(plumbing.Hash(id.RawValue())) - if err != nil { - return nil, err - } - - tree := NewTree(repo, id) - tree.gogitTree = gogitTree - return tree, nil -} - -// GetTree find the tree object in the repository. -func (repo *Repository) GetTree(idStr string) (*Tree, error) { - objectFormat, err := repo.GetObjectFormat() - if err != nil { - return nil, err - } - - if len(idStr) != objectFormat.FullLength() { - res, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(idStr).RunStdString(&RunOpts{Dir: repo.Path}) - if err != nil { - return nil, err - } - if len(res) > 0 { - idStr = res[:len(res)-1] - } - } - id, err := NewIDFromString(idStr) - if err != nil { - return nil, err - } - resolvedID := id - commitObject, err := repo.gogitRepo.CommitObject(plumbing.Hash(id.RawValue())) - if err == nil { - id = ParseGogitHash(commitObject.TreeHash) - } - treeObject, err := repo.getTree(id) - if err != nil { - return nil, err - } - treeObject.ResolvedID = resolvedID - return treeObject, nil -} diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go deleted file mode 100644 index e82012de6f..0000000000 --- a/modules/git/repo_tree_nogogit.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "io" -) - -func (repo *Repository) getTree(id ObjectID) (*Tree, error) { - wr, rd, cancel := repo.CatFileBatch(repo.Ctx) - defer cancel() - - _, _ = wr.Write([]byte(id.String() + "\n")) - - // ignore the SHA - _, typ, size, err := ReadBatchLine(rd) - if err != nil { - return nil, err - } - - switch typ { - case "tag": - resolvedID := id - data, err := io.ReadAll(io.LimitReader(rd, size)) - if err != nil { - return nil, err - } - tag, err := parseTagData(id.Type(), data) - if err != nil { - return nil, err - } - commit, err := tag.Commit(repo) - if err != nil { - return nil, err - } - commit.Tree.ResolvedID = resolvedID - return &commit.Tree, nil - case "commit": - commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size)) - if err != nil { - return nil, err - } - if _, err := rd.Discard(1); err != nil { - return nil, err - } - commit.Tree.ResolvedID = commit.ID - return &commit.Tree, nil - case "tree": - tree := NewTree(repo, id) - tree.ResolvedID = id - objectFormat, err := repo.GetObjectFormat() - if err != nil { - return nil, err - } - tree.entries, err = catBatchParseTreeEntries(objectFormat, tree, rd, size) - if err != nil { - return nil, err - } - tree.entriesParsed = true - return tree, nil - default: - if err := DiscardFull(rd, size+1); err != nil { - return nil, err - } - return nil, ErrNotExist{ - ID: id.String(), - } - } -} - -// GetTree find the tree object in the repository. -func (repo *Repository) GetTree(idStr string) (*Tree, error) { - objectFormat, err := repo.GetObjectFormat() - if err != nil { - return nil, err - } - if len(idStr) != objectFormat.FullLength() { - res, err := repo.GetRefCommitID(idStr) - if err != nil { - return nil, err - } - if len(res) > 0 { - idStr = res - } - } - id, err := NewIDFromString(idStr) - if err != nil { - return nil, err - } - - return repo.getTree(id) -} diff --git a/modules/git/signature.go b/modules/git/signature.go index f50a097758..c368ce345c 100644 --- a/modules/git/signature.go +++ b/modules/git/signature.go @@ -5,13 +5,31 @@ package git import ( + "fmt" "strconv" "strings" "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" ) +// Signature represents the Author, Committer or Tagger information. +type Signature struct { + Name string // the committer name, it can be anything + Email string // the committer email, it can be anything + When time.Time // the timestamp of the signature +} + +func (s *Signature) String() string { + return fmt.Sprintf("%s <%s>", s.Name, s.Email) +} + +// Decode decodes a byte array representing a signature to signature +func (s *Signature) Decode(b []byte) { + *s = *parseSignatureFromCommitLine(util.UnsafeBytesToString(b)) +} + // Helper to get a signature from the commit line, which looks like: // // full name 1378823654 +0200 diff --git a/modules/git/signature_gogit.go b/modules/git/signature_gogit.go deleted file mode 100644 index 1fc6aabceb..0000000000 --- a/modules/git/signature_gogit.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "github.com/go-git/go-git/v5/plumbing/object" -) - -// Signature represents the Author or Committer information. -type Signature = object.Signature diff --git a/modules/git/signature_nogogit.go b/modules/git/signature_nogogit.go deleted file mode 100644 index 0d19c0abdc..0000000000 --- a/modules/git/signature_nogogit.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "fmt" - "time" - - "code.gitea.io/gitea/modules/util" -) - -// Signature represents the Author, Committer or Tagger information. -type Signature struct { - Name string // the committer name, it can be anything - Email string // the committer email, it can be anything - When time.Time // the timestamp of the signature -} - -func (s *Signature) String() string { - return fmt.Sprintf("%s <%s>", s.Name, s.Email) -} - -// Decode decodes a byte array representing a signature to signature -func (s *Signature) Decode(b []byte) { - *s = *parseSignatureFromCommitLine(util.UnsafeBytesToString(b)) -} diff --git a/modules/git/tree.go b/modules/git/tree.go index 1da4a9fa5d..422fe68caa 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -6,9 +6,26 @@ package git import ( "bytes" + "io" "strings" ) +// Tree represents a flat directory listing. +type Tree struct { + ID ObjectID + ResolvedID ObjectID + repo *Repository + + // parent tree + ptree *Tree + + entries Entries + entriesParsed bool + + entriesRecursive Entries + entriesRecursiveParsed bool +} + // NewTree create a new tree according the repository and tree id func NewTree(repo *Repository, id ObjectID) *Tree { return &Tree{ @@ -17,6 +34,100 @@ func NewTree(repo *Repository, id ObjectID) *Tree { } } +// ListEntries returns all entries of current tree. +func (t *Tree) ListEntries() (Entries, error) { + if t.entriesParsed { + return t.entries, nil + } + + if t.repo != nil { + wr, rd, cancel := t.repo.CatFileBatch(t.repo.Ctx) + defer cancel() + + _, _ = wr.Write([]byte(t.ID.String() + "\n")) + _, typ, sz, err := ReadBatchLine(rd) + if err != nil { + return nil, err + } + if typ == "commit" { + treeID, err := ReadTreeID(rd, sz) + if err != nil && err != io.EOF { + return nil, err + } + _, _ = wr.Write([]byte(treeID + "\n")) + _, typ, sz, err = ReadBatchLine(rd) + if err != nil { + return nil, err + } + } + if typ == "tree" { + t.entries, err = catBatchParseTreeEntries(t.ID.Type(), t, rd, sz) + if err != nil { + return nil, err + } + t.entriesParsed = true + return t.entries, nil + } + + // Not a tree just use ls-tree instead + if err := DiscardFull(rd, sz+1); err != nil { + return nil, err + } + } + + stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-l").AddDynamicArguments(t.ID.String()).RunStdBytes(&RunOpts{Dir: t.repo.Path}) + if runErr != nil { + if strings.Contains(runErr.Error(), "fatal: Not a valid object name") || strings.Contains(runErr.Error(), "fatal: not a tree object") { + return nil, ErrNotExist{ + ID: t.ID.String(), + } + } + return nil, runErr + } + + var err error + t.entries, err = parseTreeEntries(stdout, t) + if err == nil { + t.entriesParsed = true + } + + return t.entries, err +} + +// listEntriesRecursive returns all entries of current tree recursively including all subtrees +// extraArgs could be "-l" to get the size, which is slower +func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) { + if t.entriesRecursiveParsed { + return t.entriesRecursive, nil + } + + stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). + AddArguments(extraArgs...). + AddDynamicArguments(t.ID.String()). + RunStdBytes(&RunOpts{Dir: t.repo.Path}) + if runErr != nil { + return nil, runErr + } + + var err error + t.entriesRecursive, err = parseTreeEntries(stdout, t) + if err == nil { + t.entriesRecursiveParsed = true + } + + return t.entriesRecursive, err +} + +// ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size +func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { + return t.listEntriesRecursive(nil) +} + +// ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees, with size +func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { + return t.listEntriesRecursive(TrustedCmdArgs{"--long"}) +} + // SubTree get a sub tree by the sub dir path func (t *Tree) SubTree(rpath string) (*Tree, error) { if len(rpath) == 0 { diff --git a/modules/git/tree_blob.go b/modules/git/tree_blob.go index e60c1f915b..df339f64b1 100644 --- a/modules/git/tree_blob.go +++ b/modules/git/tree_blob.go @@ -5,7 +5,48 @@ package git -import "strings" +import ( + "path" + "strings" +) + +// GetTreeEntryByPath get the tree entries according the sub dir +func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { + if len(relpath) == 0 { + return &TreeEntry{ + ptree: t, + ID: t.ID, + name: "", + fullName: "", + entryMode: EntryModeTree, + }, nil + } + + // FIXME: This should probably use git cat-file --batch to be a bit more efficient + relpath = path.Clean(relpath) + parts := strings.Split(relpath, "/") + var err error + tree := t + for i, name := range parts { + if i == len(parts)-1 { + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil + } + } + } else { + tree, err = tree.SubTree(name) + if err != nil { + return nil, err + } + } + } + return nil, ErrNotExist{"", relpath} +} // GetBlobByPath get the blob object according the path func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) { diff --git a/modules/git/tree_blob_gogit.go b/modules/git/tree_blob_gogit.go deleted file mode 100644 index 92c25cb92c..0000000000 --- a/modules/git/tree_blob_gogit.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "path" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// GetTreeEntryByPath get the tree entries according the sub dir -func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { - if len(relpath) == 0 { - return &TreeEntry{ - ID: t.ID, - // Type: ObjectTree, - gogitTreeEntry: &object.TreeEntry{ - Name: "", - Mode: filemode.Dir, - Hash: plumbing.Hash(t.ID.RawValue()), - }, - }, nil - } - - relpath = path.Clean(relpath) - parts := strings.Split(relpath, "/") - var err error - tree := t - for i, name := range parts { - if i == len(parts)-1 { - entries, err := tree.ListEntries() - if err != nil { - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - RelPath: relpath, - } - } - return nil, err - } - for _, v := range entries { - if v.Name() == name { - return v, nil - } - } - } else { - tree, err = tree.SubTree(name) - if err != nil { - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - RelPath: relpath, - } - } - return nil, err - } - } - } - return nil, ErrNotExist{"", relpath} -} diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go deleted file mode 100644 index 92d3d107a7..0000000000 --- a/modules/git/tree_blob_nogogit.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "path" - "strings" -) - -// GetTreeEntryByPath get the tree entries according the sub dir -func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { - if len(relpath) == 0 { - return &TreeEntry{ - ptree: t, - ID: t.ID, - name: "", - fullName: "", - entryMode: EntryModeTree, - }, nil - } - - // FIXME: This should probably use git cat-file --batch to be a bit more efficient - relpath = path.Clean(relpath) - parts := strings.Split(relpath, "/") - var err error - tree := t - for i, name := range parts { - if i == len(parts)-1 { - entries, err := tree.ListEntries() - if err != nil { - return nil, err - } - for _, v := range entries { - if v.Name() == name { - return v, nil - } - } - } else { - tree, err = tree.SubTree(name) - if err != nil { - return nil, err - } - } - } - return nil, ErrNotExist{"", relpath} -} diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index 2c47c8858c..b5dd801309 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -8,8 +8,98 @@ import ( "io" "sort" "strings" + + "code.gitea.io/gitea/modules/log" ) +// TreeEntry the leaf in the git tree +type TreeEntry struct { + ID ObjectID + + ptree *Tree + + entryMode EntryMode + name string + + size int64 + sized bool + fullName string +} + +// Name returns the name of the entry +func (te *TreeEntry) Name() string { + if te.fullName != "" { + return te.fullName + } + return te.name +} + +// Mode returns the mode of the entry +func (te *TreeEntry) Mode() EntryMode { + return te.entryMode +} + +// Size returns the size of the entry +func (te *TreeEntry) Size() int64 { + if te.IsDir() { + return 0 + } else if te.sized { + return te.size + } + + wr, rd, cancel := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx) + defer cancel() + _, err := wr.Write([]byte(te.ID.String() + "\n")) + if err != nil { + log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) + return 0 + } + _, _, te.size, err = ReadBatchLine(rd) + if err != nil { + log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) + return 0 + } + + te.sized = true + return te.size +} + +// IsSubModule if the entry is a sub module +func (te *TreeEntry) IsSubModule() bool { + return te.entryMode == EntryModeCommit +} + +// IsDir if the entry is a sub dir +func (te *TreeEntry) IsDir() bool { + return te.entryMode == EntryModeTree +} + +// IsLink if the entry is a symlink +func (te *TreeEntry) IsLink() bool { + return te.entryMode == EntryModeSymlink +} + +// IsRegular if the entry is a regular file +func (te *TreeEntry) IsRegular() bool { + return te.entryMode == EntryModeBlob +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (te *TreeEntry) IsExecutable() bool { + return te.entryMode == EntryModeExec +} + +// Blob returns the blob object the entry +func (te *TreeEntry) Blob() *Blob { + return &Blob{ + ID: te.ID, + name: te.Name(), + size: te.size, + gotSize: te.sized, + repo: te.ptree.repo, + } +} + // Type returns the type of the entry (commit, tree, blob) func (te *TreeEntry) Type() string { switch te.Mode() { diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go deleted file mode 100644 index eb9b012681..0000000000 --- a/modules/git/tree_entry_gogit.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// TreeEntry the leaf in the git tree -type TreeEntry struct { - ID ObjectID - - gogitTreeEntry *object.TreeEntry - ptree *Tree - - size int64 - sized bool - fullName string -} - -// Name returns the name of the entry -func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } - return te.gogitTreeEntry.Name -} - -// Mode returns the mode of the entry -func (te *TreeEntry) Mode() EntryMode { - return EntryMode(te.gogitTreeEntry.Mode) -} - -// Size returns the size of the entry -func (te *TreeEntry) Size() int64 { - if te.IsDir() { - return 0 - } else if te.sized { - return te.size - } - - file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry) - if err != nil { - return 0 - } - - te.sized = true - te.size = file.Size - return te.size -} - -// IsSubModule if the entry is a sub module -func (te *TreeEntry) IsSubModule() bool { - return te.gogitTreeEntry.Mode == filemode.Submodule -} - -// IsDir if the entry is a sub dir -func (te *TreeEntry) IsDir() bool { - return te.gogitTreeEntry.Mode == filemode.Dir -} - -// IsLink if the entry is a symlink -func (te *TreeEntry) IsLink() bool { - return te.gogitTreeEntry.Mode == filemode.Symlink -} - -// IsRegular if the entry is a regular file -func (te *TreeEntry) IsRegular() bool { - return te.gogitTreeEntry.Mode == filemode.Regular -} - -// IsExecutable if the entry is an executable file (not necessarily binary) -func (te *TreeEntry) IsExecutable() bool { - return te.gogitTreeEntry.Mode == filemode.Executable -} - -// Blob returns the blob object the entry -func (te *TreeEntry) Blob() *Blob { - encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash) - if err != nil { - return nil - } - - return &Blob{ - ID: ParseGogitHash(te.gogitTreeEntry.Hash), - gogitEncodedObj: encodedObj, - name: te.Name(), - } -} diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go deleted file mode 100644 index 89244e27ee..0000000000 --- a/modules/git/tree_entry_nogogit.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import "code.gitea.io/gitea/modules/log" - -// TreeEntry the leaf in the git tree -type TreeEntry struct { - ID ObjectID - - ptree *Tree - - entryMode EntryMode - name string - - size int64 - sized bool - fullName string -} - -// Name returns the name of the entry -func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } - return te.name -} - -// Mode returns the mode of the entry -func (te *TreeEntry) Mode() EntryMode { - return te.entryMode -} - -// Size returns the size of the entry -func (te *TreeEntry) Size() int64 { - if te.IsDir() { - return 0 - } else if te.sized { - return te.size - } - - wr, rd, cancel := te.ptree.repo.CatFileBatchCheck(te.ptree.repo.Ctx) - defer cancel() - _, err := wr.Write([]byte(te.ID.String() + "\n")) - if err != nil { - log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) - return 0 - } - _, _, te.size, err = ReadBatchLine(rd) - if err != nil { - log.Debug("error whilst reading size for %s in %s. Error: %v", te.ID.String(), te.ptree.repo.Path, err) - return 0 - } - - te.sized = true - return te.size -} - -// IsSubModule if the entry is a sub module -func (te *TreeEntry) IsSubModule() bool { - return te.entryMode == EntryModeCommit -} - -// IsDir if the entry is a sub dir -func (te *TreeEntry) IsDir() bool { - return te.entryMode == EntryModeTree -} - -// IsLink if the entry is a symlink -func (te *TreeEntry) IsLink() bool { - return te.entryMode == EntryModeSymlink -} - -// IsRegular if the entry is a regular file -func (te *TreeEntry) IsRegular() bool { - return te.entryMode == EntryModeBlob -} - -// IsExecutable if the entry is an executable file (not necessarily binary) -func (te *TreeEntry) IsExecutable() bool { - return te.entryMode == EntryModeExec -} - -// Blob returns the blob object the entry -func (te *TreeEntry) Blob() *Blob { - return &Blob{ - ID: te.ID, - name: te.Name(), - size: te.size, - gotSize: te.sized, - repo: te.ptree.repo, - } -} diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go deleted file mode 100644 index e628c05a82..0000000000 --- a/modules/git/tree_entry_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "testing" - - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func getTestEntries() Entries { - return Entries{ - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v1.0", Mode: filemode.Dir}}, - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.0", Mode: filemode.Dir}}, - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.1", Mode: filemode.Dir}}, - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.12", Mode: filemode.Dir}}, - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v2.2", Mode: filemode.Dir}}, - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "v12.0", Mode: filemode.Dir}}, - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "abc", Mode: filemode.Regular}}, - &TreeEntry{gogitTreeEntry: &object.TreeEntry{Name: "bcd", Mode: filemode.Regular}}, - } -} - -func TestEntriesSort(t *testing.T) { - entries := getTestEntries() - entries.Sort() - assert.Equal(t, "v1.0", entries[0].Name()) - assert.Equal(t, "v12.0", entries[1].Name()) - assert.Equal(t, "v2.0", entries[2].Name()) - assert.Equal(t, "v2.1", entries[3].Name()) - assert.Equal(t, "v2.12", entries[4].Name()) - assert.Equal(t, "v2.2", entries[5].Name()) - assert.Equal(t, "abc", entries[6].Name()) - assert.Equal(t, "bcd", entries[7].Name()) -} - -func TestEntriesCustomSort(t *testing.T) { - entries := getTestEntries() - entries.CustomSort(func(s1, s2 string) bool { - return s1 > s2 - }) - assert.Equal(t, "v2.2", entries[0].Name()) - assert.Equal(t, "v2.12", entries[1].Name()) - assert.Equal(t, "v2.1", entries[2].Name()) - assert.Equal(t, "v2.0", entries[3].Name()) - assert.Equal(t, "v12.0", entries[4].Name()) - assert.Equal(t, "v1.0", entries[5].Name()) - assert.Equal(t, "bcd", entries[6].Name()) - assert.Equal(t, "abc", entries[7].Name()) -} - -func TestFollowLink(t *testing.T) { - r, err := openRepositoryWithDefaultContext("tests/repos/repo1_bare") - require.NoError(t, err) - defer r.Close() - - commit, err := r.GetCommit("37991dec2c8e592043f47155ce4808d4580f9123") - require.NoError(t, err) - - // get the symlink - lnk, err := commit.Tree.GetTreeEntryByPath("foo/bar/link_to_hello") - require.NoError(t, err) - assert.True(t, lnk.IsLink()) - - // should be able to dereference to target - target, err := lnk.FollowLink() - require.NoError(t, err) - assert.Equal(t, "hello", target.Name()) - assert.False(t, target.IsLink()) - assert.Equal(t, "b14df6442ea5a1b382985a6549b85d435376c351", target.ID.String()) - - // should error when called on normal file - target, err = commit.Tree.GetTreeEntryByPath("file1.txt") - require.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "file1.txt: not a symlink") - - // should error for broken links - target, err = commit.Tree.GetTreeEntryByPath("foo/broken_link") - require.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "broken_link: broken link") - - // should error for external links - target, err = commit.Tree.GetTreeEntryByPath("foo/outside_repo") - require.NoError(t, err) - assert.True(t, target.IsLink()) - _, err = target.FollowLink() - assert.EqualError(t, err, "outside_repo: points outside of repo") - - // testing fix for short link bug - target, err = commit.Tree.GetTreeEntryByPath("foo/link_short") - require.NoError(t, err) - _, err = target.FollowLink() - assert.EqualError(t, err, "link_short: broken link") -} diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go deleted file mode 100644 index 421b0ecb0f..0000000000 --- a/modules/git/tree_gogit.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package git - -import ( - "io" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// Tree represents a flat directory listing. -type Tree struct { - ID ObjectID - ResolvedID ObjectID - repo *Repository - - gogitTree *object.Tree - - // parent tree - ptree *Tree -} - -func (t *Tree) loadTreeObject() error { - gogitTree, err := t.repo.gogitRepo.TreeObject(plumbing.Hash(t.ID.RawValue())) - if err != nil { - return err - } - - t.gogitTree = gogitTree - return nil -} - -// ListEntries returns all entries of current tree. -func (t *Tree) ListEntries() (Entries, error) { - if t.gogitTree == nil { - err := t.loadTreeObject() - if err != nil { - return nil, err - } - } - - entries := make([]*TreeEntry, len(t.gogitTree.Entries)) - for i, entry := range t.gogitTree.Entries { - entries[i] = &TreeEntry{ - ID: ParseGogitHash(entry.Hash), - gogitTreeEntry: &t.gogitTree.Entries[i], - ptree: t, - } - } - - return entries, nil -} - -// ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees -func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { - if t.gogitTree == nil { - err := t.loadTreeObject() - if err != nil { - return nil, err - } - } - - var entries []*TreeEntry - seen := map[plumbing.Hash]bool{} - walker := object.NewTreeWalker(t.gogitTree, true, seen) - for { - fullName, entry, err := walker.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if seen[entry.Hash] { - continue - } - - convertedEntry := &TreeEntry{ - ID: ParseGogitHash(entry.Hash), - gogitTreeEntry: &entry, - ptree: t, - fullName: fullName, - } - entries = append(entries, convertedEntry) - } - - return entries, nil -} - -// ListEntriesRecursiveFast is the alias of ListEntriesRecursiveWithSize for the gogit version -func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { - return t.ListEntriesRecursiveWithSize() -} diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go deleted file mode 100644 index e0a72de5b8..0000000000 --- a/modules/git/tree_nogogit.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build !gogit - -package git - -import ( - "io" - "strings" -) - -// Tree represents a flat directory listing. -type Tree struct { - ID ObjectID - ResolvedID ObjectID - repo *Repository - - // parent tree - ptree *Tree - - entries Entries - entriesParsed bool - - entriesRecursive Entries - entriesRecursiveParsed bool -} - -// ListEntries returns all entries of current tree. -func (t *Tree) ListEntries() (Entries, error) { - if t.entriesParsed { - return t.entries, nil - } - - if t.repo != nil { - wr, rd, cancel := t.repo.CatFileBatch(t.repo.Ctx) - defer cancel() - - _, _ = wr.Write([]byte(t.ID.String() + "\n")) - _, typ, sz, err := ReadBatchLine(rd) - if err != nil { - return nil, err - } - if typ == "commit" { - treeID, err := ReadTreeID(rd, sz) - if err != nil && err != io.EOF { - return nil, err - } - _, _ = wr.Write([]byte(treeID + "\n")) - _, typ, sz, err = ReadBatchLine(rd) - if err != nil { - return nil, err - } - } - if typ == "tree" { - t.entries, err = catBatchParseTreeEntries(t.ID.Type(), t, rd, sz) - if err != nil { - return nil, err - } - t.entriesParsed = true - return t.entries, nil - } - - // Not a tree just use ls-tree instead - if err := DiscardFull(rd, sz+1); err != nil { - return nil, err - } - } - - stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-l").AddDynamicArguments(t.ID.String()).RunStdBytes(&RunOpts{Dir: t.repo.Path}) - if runErr != nil { - if strings.Contains(runErr.Error(), "fatal: Not a valid object name") || strings.Contains(runErr.Error(), "fatal: not a tree object") { - return nil, ErrNotExist{ - ID: t.ID.String(), - } - } - return nil, runErr - } - - var err error - t.entries, err = parseTreeEntries(stdout, t) - if err == nil { - t.entriesParsed = true - } - - return t.entries, err -} - -// listEntriesRecursive returns all entries of current tree recursively including all subtrees -// extraArgs could be "-l" to get the size, which is slower -func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) { - if t.entriesRecursiveParsed { - return t.entriesRecursive, nil - } - - stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). - AddArguments(extraArgs...). - AddDynamicArguments(t.ID.String()). - RunStdBytes(&RunOpts{Dir: t.repo.Path}) - if runErr != nil { - return nil, runErr - } - - var err error - t.entriesRecursive, err = parseTreeEntries(stdout, t) - if err == nil { - t.entriesRecursiveParsed = true - } - - return t.entriesRecursive, err -} - -// ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size -func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { - return t.listEntriesRecursive(nil) -} - -// ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees, with size -func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { - return t.listEntriesRecursive(TrustedCmdArgs{"--long"}) -} diff --git a/modules/git/utils_test.go b/modules/git/utils_test.go index a3c2b7f8eb..a8c3fe38f6 100644 --- a/modules/git/utils_test.go +++ b/modules/git/utils_test.go @@ -13,7 +13,7 @@ import ( // but not in production code. func skipIfSHA256NotSupported(t *testing.T) { - if isGogit || CheckGitVersionAtLeast("2.42") != nil { + if CheckGitVersionAtLeast("2.42") != nil { t.Skip("skipping because installed Git version doesn't support SHA256") } } diff --git a/modules/gitrepo/walk_nogogit.go b/modules/gitrepo/walk.go similarity index 95% rename from modules/gitrepo/walk_nogogit.go rename to modules/gitrepo/walk.go index ff9555996d..8c672ea78b 100644 --- a/modules/gitrepo/walk_nogogit.go +++ b/modules/gitrepo/walk.go @@ -1,8 +1,6 @@ // Copyright 2024 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build !gogit - package gitrepo import ( diff --git a/modules/gitrepo/walk_gogit.go b/modules/gitrepo/walk_gogit.go deleted file mode 100644 index 6370faf08e..0000000000 --- a/modules/gitrepo/walk_gogit.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package gitrepo - -import ( - "context" - - "github.com/go-git/go-git/v5/plumbing" -) - -// WalkReferences walks all the references from the repository -// refname is empty, ObjectTag or ObjectBranch. All other values should be treated as equivalent to empty. -func WalkReferences(ctx context.Context, repo Repository, walkfn func(sha1, refname string) error) (int, error) { - gitRepo := repositoryFromContext(ctx, repo) - if gitRepo == nil { - var err error - gitRepo, err = OpenRepository(ctx, repo) - if err != nil { - return 0, err - } - defer gitRepo.Close() - } - - i := 0 - iter, err := gitRepo.GoGitRepo().References() - if err != nil { - return i, err - } - defer iter.Close() - - err = iter.ForEach(func(ref *plumbing.Reference) error { - err := walkfn(ref.Hash().String(), string(ref.Name())) - i++ - return err - }) - return i, err -} diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner.go similarity index 99% rename from modules/lfs/pointer_scanner_nogogit.go rename to modules/lfs/pointer_scanner.go index 658b98feab..8bbf7a8692 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner.go @@ -1,8 +1,6 @@ // Copyright 2021 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build !gogit - package lfs import ( diff --git a/modules/lfs/pointer_scanner_gogit.go b/modules/lfs/pointer_scanner_gogit.go deleted file mode 100644 index f4302c23bc..0000000000 --- a/modules/lfs/pointer_scanner_gogit.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -//go:build gogit - -package lfs - -import ( - "context" - "fmt" - - "code.gitea.io/gitea/modules/git" - - "github.com/go-git/go-git/v5/plumbing/object" -) - -// SearchPointerBlobs scans the whole repository for LFS pointer files -func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) { - gitRepo := repo.GoGitRepo() - - err := func() error { - blobs, err := gitRepo.BlobObjects() - if err != nil { - return fmt.Errorf("lfs.SearchPointerBlobs BlobObjects: %w", err) - } - - return blobs.ForEach(func(blob *object.Blob) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - if blob.Size > blobSizeCutoff { - return nil - } - - reader, err := blob.Reader() - if err != nil { - return fmt.Errorf("lfs.SearchPointerBlobs blob.Reader: %w", err) - } - defer reader.Close() - - pointer, _ := ReadPointer(reader) - if pointer.IsValid() { - pointerChan <- PointerBlob{Hash: blob.Hash.String(), Pointer: pointer} - } - - return nil - }) - }() - if err != nil { - select { - case <-ctx.Done(): - default: - errChan <- err - } - } - - close(pointerChan) - close(errChan) -} diff --git a/release-notes/4941.md b/release-notes/4941.md new file mode 100644 index 0000000000..85b896a8d3 --- /dev/null +++ b/release-notes/4941.md @@ -0,0 +1 @@ +Drop support to build Forgejo with the optional go-git Git backend. It only affects users who built Forgejo manually using `TAGS=gogits`, which no longer has any effect. Moving forward, we only support the default backend using the git binary. Please get in touch if you used the go-git backend and require any assistance moving away from it.