diff options
Diffstat (limited to '')
| -rw-r--r-- | modules/repository/repo.go | 383 |
1 files changed, 383 insertions, 0 deletions
diff --git a/modules/repository/repo.go b/modules/repository/repo.go new file mode 100644 index 00000000..a863bec9 --- /dev/null +++ b/modules/repository/repo.go @@ -0,0 +1,383 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2024 The Forgejo Authors c/o Codeberg e.V.. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "code.gitea.io/gitea/models/db" + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" +) + +/* +GitHub, GitLab, Gogs: *.wiki.git +BitBucket: *.git/wiki +*/ +var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"} + +// WikiRemoteURL returns accessible repository URL for wiki if exists. +// Otherwise, it returns an empty string. +func WikiRemoteURL(ctx context.Context, remote string) string { + remote = strings.TrimSuffix(remote, ".git") + for _, suffix := range commonWikiURLSuffixes { + wikiURL := remote + suffix + if git.IsRepoURLAccessible(ctx, wikiURL) { + return wikiURL + } + } + return "" +} + +// SyncRepoTags synchronizes releases table with repository tags +func SyncRepoTags(ctx context.Context, repoID int64) error { + repo, err := repo_model.GetRepositoryByID(ctx, repoID) + if err != nil { + return err + } + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + return err + } + defer gitRepo.Close() + + return SyncReleasesWithTags(ctx, repo, gitRepo) +} + +// SyncReleasesWithTags synchronizes release table with repository tags +func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + + // optimized procedure for pull-mirrors which saves a lot of time (in + // particular for repos with many tags). + if repo.IsMirror { + return pullMirrorReleaseSync(ctx, repo, gitRepo) + } + + existingRelTags := make(container.Set[string]) + opts := repo_model.FindReleasesOptions{ + IncludeDrafts: true, + IncludeTags: true, + ListOptions: db.ListOptions{PageSize: 50}, + RepoID: repo.ID, + } + for page := 1; ; page++ { + opts.Page = page + rels, err := db.Find[repo_model.Release](gitRepo.Ctx, opts) + if err != nil { + return fmt.Errorf("unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + if len(rels) == 0 { + break + } + for _, rel := range rels { + if rel.IsDraft { + continue + } + commitID, err := gitRepo.GetTagCommitID(rel.TagName) + if err != nil && !git.IsErrNotExist(err) { + return fmt.Errorf("unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) + } + if git.IsErrNotExist(err) || commitID != rel.Sha1 { + if err := repo_model.PushUpdateDeleteTag(ctx, repo, rel.TagName); err != nil { + return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err) + } + } else { + existingRelTags.Add(strings.ToLower(rel.TagName)) + } + } + } + + _, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error { + tagName := strings.TrimPrefix(refname, git.TagPrefix) + if existingRelTags.Contains(strings.ToLower(tagName)) { + return nil + } + + if err := PushUpdateAddTag(ctx, repo, gitRepo, tagName, sha1, refname); err != nil { + // sometimes, some tags will be sync failed. i.e. https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tag/?h=v2.6.11 + // this is a tree object, not a tag object which created before git + log.Error("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %v", tagName, repo.ID, repo.OwnerName, repo.Name, err) + } + + return nil + }) + return err +} + +// PushUpdateAddTag must be called for any push actions to add tag +func PushUpdateAddTag(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tagName, sha1, refname string) error { + tag, err := gitRepo.GetTagWithID(sha1, tagName) + if err != nil { + return fmt.Errorf("unable to GetTag: %w", err) + } + commit, err := tag.Commit(gitRepo) + if err != nil { + return fmt.Errorf("unable to get tag Commit: %w", err) + } + + sig := tag.Tagger + if sig == nil { + sig = commit.Author + } + if sig == nil { + sig = commit.Committer + } + + var author *user_model.User + createdAt := time.Unix(1, 0) + + if sig != nil { + author, err = user_model.GetUserByEmail(ctx, sig.Email) + if err != nil && !user_model.IsErrUserNotExist(err) { + return fmt.Errorf("unable to GetUserByEmail for %q: %w", sig.Email, err) + } + createdAt = sig.When + } + + commitsCount, err := commit.CommitsCount() + if err != nil { + return fmt.Errorf("unable to get CommitsCount: %w", err) + } + + rel := repo_model.Release{ + RepoID: repo.ID, + TagName: tagName, + LowerTagName: strings.ToLower(tagName), + Sha1: commit.ID.String(), + NumCommits: commitsCount, + CreatedUnix: timeutil.TimeStamp(createdAt.Unix()), + IsTag: true, + } + if author != nil { + rel.PublisherID = author.ID + } + + return repo_model.SaveOrUpdateTag(ctx, repo, &rel) +} + +// StoreMissingLfsObjectsInRepository downloads missing LFS objects +func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error { + contentStore := lfs.NewContentStore() + + pointerChan := make(chan lfs.PointerBlob) + errChan := make(chan error, 1) + go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) + + downloadObjects := func(pointers []lfs.Pointer) error { + err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { + if objectError != nil { + return objectError + } + + defer content.Close() + + _, err := git_model.NewLFSMetaObject(ctx, repo.ID, p) + if err != nil { + log.Error("Repo[%-v]: Error creating LFS meta object %-v: %v", repo, p, err) + return err + } + + if err := contentStore.Put(p, content); err != nil { + log.Error("Repo[%-v]: Error storing content for LFS meta object %-v: %v", repo, p, err) + if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repo.ID, p.Oid); err2 != nil { + log.Error("Repo[%-v]: Error removing LFS meta object %-v: %v", repo, p, err2) + } + return err + } + return nil + }) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + } + return err + } + + var batch []lfs.Pointer + for pointerBlob := range pointerChan { + meta, err := git_model.GetLFSMetaObjectByOid(ctx, repo.ID, pointerBlob.Oid) + if err != nil && err != git_model.ErrLFSObjectNotExist { + log.Error("Repo[%-v]: Error querying LFS meta object %-v: %v", repo, pointerBlob.Pointer, err) + return err + } + if meta != nil { + log.Trace("Repo[%-v]: Skipping unknown LFS meta object %-v", repo, pointerBlob.Pointer) + continue + } + + log.Trace("Repo[%-v]: LFS object %-v not present in repository", repo, pointerBlob.Pointer) + + exist, err := contentStore.Exists(pointerBlob.Pointer) + if err != nil { + log.Error("Repo[%-v]: Error checking if LFS object %-v exists: %v", repo, pointerBlob.Pointer, err) + return err + } + + if exist { + log.Trace("Repo[%-v]: LFS object %-v already present; creating meta object", repo, pointerBlob.Pointer) + _, err := git_model.NewLFSMetaObject(ctx, repo.ID, pointerBlob.Pointer) + if err != nil { + log.Error("Repo[%-v]: Error creating LFS meta object %-v: %v", repo, pointerBlob.Pointer, err) + return err + } + } else { + if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { + log.Info("Repo[%-v]: LFS object %-v download denied because of LFS_MAX_FILE_SIZE=%d < size %d", repo, pointerBlob.Pointer, setting.LFS.MaxFileSize, pointerBlob.Size) + continue + } + + batch = append(batch, pointerBlob.Pointer) + if len(batch) >= lfsClient.BatchSize() { + if err := downloadObjects(batch); err != nil { + return err + } + batch = nil + } + } + } + if len(batch) > 0 { + if err := downloadObjects(batch); err != nil { + return err + } + } + + err, has := <-errChan + if has { + log.Error("Repo[%-v]: Error enumerating LFS objects for repository: %v", repo, err) + return err + } + + return nil +} + +// shortRelease to reduce load memory, this struct can replace repo_model.Release +type shortRelease struct { + ID int64 + TagName string + Sha1 string + IsTag bool +} + +func (shortRelease) TableName() string { + return "release" +} + +// pullMirrorReleaseSync is a pull-mirror specific tag<->release table +// synchronization which overwrites all Releases from the repository tags. This +// can be relied on since a pull-mirror is always identical to its +// upstream. Hence, after each sync we want the pull-mirror release set to be +// identical to the upstream tag set. This is much more efficient for +// repositories like https://github.com/vim/vim (with over 13000 tags). +func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + tags, numTags, err := gitRepo.GetTagInfos(0, 0) + if err != nil { + return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + err = db.WithTx(ctx, func(ctx context.Context) error { + dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{ + RepoID: repo.ID, + IncludeDrafts: true, + IncludeTags: true, + }) + if err != nil { + return fmt.Errorf("unable to FindReleases in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + + inserts, deletes, updates := calcSync(tags, dbReleases) + // + // make release set identical to upstream tags + // + for _, tag := range inserts { + release := repo_model.Release{ + RepoID: repo.ID, + TagName: tag.Name, + LowerTagName: strings.ToLower(tag.Name), + Sha1: tag.Object.String(), + // NOTE: ignored, since NumCommits are unused + // for pull-mirrors (only relevant when + // displaying releases, IsTag: false) + NumCommits: -1, + CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()), + IsTag: true, + } + if err := db.Insert(ctx, release); err != nil { + return fmt.Errorf("unable insert tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) + } + } + + // only delete tags releases + if len(deletes) > 0 { + if _, err := db.GetEngine(ctx).Where("repo_id=?", repo.ID). + In("id", deletes). + Delete(&repo_model.Release{}); err != nil { + return fmt.Errorf("unable to delete tags for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + } + + for _, tag := range updates { + if _, err := db.GetEngine(ctx).Where("repo_id = ? AND lower_tag_name = ?", repo.ID, strings.ToLower(tag.Name)). + Cols("sha1"). + Update(&repo_model.Release{ + Sha1: tag.Object.String(), + }); err != nil { + return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + + log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags) + return nil +} + +func calcSync(destTags []*git.Tag, dbTags []*shortRelease) ([]*git.Tag, []int64, []*git.Tag) { + destTagMap := make(map[string]*git.Tag) + for _, tag := range destTags { + destTagMap[tag.Name] = tag + } + dbTagMap := make(map[string]*shortRelease) + for _, rel := range dbTags { + dbTagMap[rel.TagName] = rel + } + + inserted := make([]*git.Tag, 0, 10) + updated := make([]*git.Tag, 0, 10) + for _, tag := range destTags { + rel := dbTagMap[tag.Name] + if rel == nil { + inserted = append(inserted, tag) + } else if rel.Sha1 != tag.Object.String() { + updated = append(updated, tag) + } + } + deleted := make([]int64, 0, 10) + for _, tag := range dbTags { + if destTagMap[tag.TagName] == nil && tag.IsTag { + deleted = append(deleted, tag.ID) + } + } + return inserted, deleted, updated +} |