gitea源码

mirror_pull.go 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652
  1. // Copyright 2021 The Gitea Authors. All rights reserved.
  2. // SPDX-License-Identifier: MIT
  3. package mirror
  4. import (
  5. "context"
  6. "fmt"
  7. "strings"
  8. "time"
  9. repo_model "code.gitea.io/gitea/models/repo"
  10. system_model "code.gitea.io/gitea/models/system"
  11. "code.gitea.io/gitea/modules/cache"
  12. "code.gitea.io/gitea/modules/git"
  13. "code.gitea.io/gitea/modules/git/gitcmd"
  14. giturl "code.gitea.io/gitea/modules/git/url"
  15. "code.gitea.io/gitea/modules/gitrepo"
  16. "code.gitea.io/gitea/modules/globallock"
  17. "code.gitea.io/gitea/modules/lfs"
  18. "code.gitea.io/gitea/modules/log"
  19. "code.gitea.io/gitea/modules/process"
  20. "code.gitea.io/gitea/modules/proxy"
  21. repo_module "code.gitea.io/gitea/modules/repository"
  22. "code.gitea.io/gitea/modules/setting"
  23. "code.gitea.io/gitea/modules/timeutil"
  24. "code.gitea.io/gitea/modules/util"
  25. notify_service "code.gitea.io/gitea/services/notify"
  26. repo_service "code.gitea.io/gitea/services/repository"
  27. )
  28. // gitShortEmptySha Git short empty SHA
  29. const gitShortEmptySha = "0000000"
  30. // UpdateAddress writes new address to Git repository and database
  31. func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error {
  32. u, err := giturl.ParseGitURL(addr)
  33. if err != nil {
  34. return fmt.Errorf("invalid addr: %v", err)
  35. }
  36. remoteName := m.GetRemoteName()
  37. repo := m.GetRepository(ctx)
  38. // Remove old remote
  39. err = gitrepo.GitRemoteRemove(ctx, repo, remoteName)
  40. if err != nil && !git.IsRemoteNotExistError(err) {
  41. return err
  42. }
  43. err = gitrepo.GitRemoteAdd(ctx, repo, remoteName, addr, gitrepo.RemoteOptionMirrorFetch)
  44. if err != nil && !git.IsRemoteNotExistError(err) {
  45. return err
  46. }
  47. if repo_service.HasWiki(ctx, m.Repo) {
  48. wikiRemotePath := repo_module.WikiRemoteURL(ctx, addr)
  49. // Remove old remote of wiki
  50. err = gitrepo.GitRemoteRemove(ctx, repo.WikiStorageRepo(), remoteName)
  51. if err != nil && !git.IsRemoteNotExistError(err) {
  52. return err
  53. }
  54. err = gitrepo.GitRemoteAdd(ctx, repo.WikiStorageRepo(), remoteName, wikiRemotePath, gitrepo.RemoteOptionMirrorFetch)
  55. if err != nil && !git.IsRemoteNotExistError(err) {
  56. return err
  57. }
  58. }
  59. // erase authentication before storing in database
  60. u.User = nil
  61. m.Repo.OriginalURL = u.String()
  62. return repo_model.UpdateRepositoryColsNoAutoTime(ctx, m.Repo, "original_url")
  63. }
  64. // mirrorSyncResult contains information of a updated reference.
  65. // If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty.
  66. // If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty.
  67. type mirrorSyncResult struct {
  68. refName git.RefName
  69. oldCommitID string
  70. newCommitID string
  71. }
  72. // parseRemoteUpdateOutput detects create, update and delete operations of references from upstream.
  73. // possible output example:
  74. /*
  75. // * [new tag] v0.1.8 -> v0.1.8
  76. // * [new branch] master -> origin/master
  77. // * [new ref] refs/pull/2/head -> refs/pull/2/head"
  78. // - [deleted] (none) -> origin/test // delete a branch
  79. // - [deleted] (none) -> 1 // delete a tag
  80. // 957a993..a87ba5f test -> origin/test
  81. // + f895a1e...957a993 test -> origin/test (forced update)
  82. */
  83. // TODO: return whether it's a force update
  84. func parseRemoteUpdateOutput(output, remoteName string) []*mirrorSyncResult {
  85. results := make([]*mirrorSyncResult, 0, 3)
  86. lines := strings.Split(output, "\n")
  87. for i := range lines {
  88. // Make sure reference name is presented before continue
  89. idx := strings.Index(lines[i], "-> ")
  90. if idx == -1 {
  91. continue
  92. }
  93. refName := strings.TrimSpace(lines[i][idx+3:])
  94. switch {
  95. case strings.HasPrefix(lines[i], " * [new tag]"): // new tag
  96. results = append(results, &mirrorSyncResult{
  97. refName: git.RefNameFromTag(refName),
  98. oldCommitID: gitShortEmptySha,
  99. })
  100. case strings.HasPrefix(lines[i], " * [new branch]"): // new branch
  101. refName = strings.TrimPrefix(refName, remoteName+"/")
  102. results = append(results, &mirrorSyncResult{
  103. refName: git.RefNameFromBranch(refName),
  104. oldCommitID: gitShortEmptySha,
  105. })
  106. case strings.HasPrefix(lines[i], " * [new ref]"): // new reference
  107. results = append(results, &mirrorSyncResult{
  108. refName: git.RefName(refName),
  109. oldCommitID: gitShortEmptySha,
  110. })
  111. case strings.HasPrefix(lines[i], " - "): // Delete reference
  112. isTag := !strings.HasPrefix(refName, remoteName+"/")
  113. var refFullName git.RefName
  114. if strings.HasPrefix(refName, "refs/") {
  115. refFullName = git.RefName(refName)
  116. } else if isTag {
  117. refFullName = git.RefNameFromTag(refName)
  118. } else {
  119. refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
  120. }
  121. results = append(results, &mirrorSyncResult{
  122. refName: refFullName,
  123. newCommitID: gitShortEmptySha,
  124. })
  125. case strings.HasPrefix(lines[i], " + "): // Force update
  126. if idx := strings.Index(refName, " "); idx > -1 {
  127. refName = refName[:idx]
  128. }
  129. delimIdx := strings.Index(lines[i][3:], " ")
  130. if delimIdx == -1 {
  131. log.Error("SHA delimiter not found: %q", lines[i])
  132. continue
  133. }
  134. shas := strings.Split(lines[i][3:delimIdx+3], "...")
  135. if len(shas) != 2 {
  136. log.Error("Expect two SHAs but not what found: %q", lines[i])
  137. continue
  138. }
  139. var refFullName git.RefName
  140. if strings.HasPrefix(refName, "refs/") {
  141. refFullName = git.RefName(refName)
  142. } else {
  143. refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
  144. }
  145. results = append(results, &mirrorSyncResult{
  146. refName: refFullName,
  147. oldCommitID: shas[0],
  148. newCommitID: shas[1],
  149. })
  150. case strings.HasPrefix(lines[i], " "): // New commits of a reference
  151. delimIdx := strings.Index(lines[i][3:], " ")
  152. if delimIdx == -1 {
  153. log.Error("SHA delimiter not found: %q", lines[i])
  154. continue
  155. }
  156. shas := strings.Split(lines[i][3:delimIdx+3], "..")
  157. if len(shas) != 2 {
  158. log.Error("Expect two SHAs but not what found: %q", lines[i])
  159. continue
  160. }
  161. var refFullName git.RefName
  162. if strings.HasPrefix(refName, "refs/") {
  163. refFullName = git.RefName(refName)
  164. } else {
  165. refFullName = git.RefNameFromBranch(strings.TrimPrefix(refName, remoteName+"/"))
  166. }
  167. results = append(results, &mirrorSyncResult{
  168. refName: refFullName,
  169. oldCommitID: shas[0],
  170. newCommitID: shas[1],
  171. })
  172. default:
  173. log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i])
  174. }
  175. }
  176. return results
  177. }
  178. func pruneBrokenReferences(ctx context.Context,
  179. m *repo_model.Mirror,
  180. timeout time.Duration,
  181. stdoutBuilder, stderrBuilder *strings.Builder,
  182. isWiki bool,
  183. ) error {
  184. wiki := ""
  185. var storageRepo gitrepo.Repository = m.Repo
  186. if isWiki {
  187. wiki = "Wiki "
  188. storageRepo = m.Repo.WikiStorageRepo()
  189. }
  190. stderrBuilder.Reset()
  191. stdoutBuilder.Reset()
  192. pruneErr := gitrepo.GitRemotePrune(ctx, storageRepo, m.GetRemoteName(), timeout, stdoutBuilder, stderrBuilder)
  193. if pruneErr != nil {
  194. stdout := stdoutBuilder.String()
  195. stderr := stderrBuilder.String()
  196. // sanitize the output, since it may contain the remote address, which may
  197. // contain a password
  198. stderrMessage := util.SanitizeCredentialURLs(stderr)
  199. stdoutMessage := util.SanitizeCredentialURLs(stdout)
  200. log.Error("Failed to prune mirror repository %s%-v references:\nStdout: %s\nStderr: %s\nErr: %v", wiki, m.Repo, stdoutMessage, stderrMessage, pruneErr)
  201. desc := fmt.Sprintf("Failed to prune mirror repository %s'%s' references: %s", wiki, storageRepo.RelativePath(), stderrMessage)
  202. if err := system_model.CreateRepositoryNotice(desc); err != nil {
  203. log.Error("CreateRepositoryNotice: %v", err)
  204. }
  205. // this if will only be reached on a successful prune so try to get the mirror again
  206. }
  207. return pruneErr
  208. }
  209. // checkRecoverableSyncError takes an error message from a git fetch command and returns false if it should be a fatal/blocking error
  210. func checkRecoverableSyncError(stderrMessage string) bool {
  211. switch {
  212. case strings.Contains(stderrMessage, "unable to resolve reference") && strings.Contains(stderrMessage, "reference broken"):
  213. return true
  214. case strings.Contains(stderrMessage, "remote error") && strings.Contains(stderrMessage, "not our ref"):
  215. return true
  216. case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "but expected"):
  217. return true
  218. case strings.Contains(stderrMessage, "cannot lock ref") && strings.Contains(stderrMessage, "unable to resolve reference"):
  219. return true
  220. case strings.Contains(stderrMessage, "Unable to create") && strings.Contains(stderrMessage, ".lock"):
  221. return true
  222. default:
  223. return false
  224. }
  225. }
  226. // runSync returns true if sync finished without error.
  227. func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bool) {
  228. repoPath := m.Repo.RepoPath()
  229. wikiPath := m.Repo.WikiPath()
  230. timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
  231. log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo)
  232. // use fetch but not remote update because git fetch support --tags but remote update doesn't
  233. cmd := gitcmd.NewCommand("fetch")
  234. if m.EnablePrune {
  235. cmd.AddArguments("--prune")
  236. }
  237. cmd.AddArguments("--tags").AddDynamicArguments(m.GetRemoteName())
  238. remoteURL, remoteErr := gitrepo.GitRemoteGetURL(ctx, m.Repo, m.GetRemoteName())
  239. if remoteErr != nil {
  240. log.Error("SyncMirrors [repo: %-v]: GetRemoteURL Error %v", m.Repo, remoteErr)
  241. return nil, false
  242. }
  243. envs := proxy.EnvWithProxy(remoteURL.URL)
  244. stdoutBuilder := strings.Builder{}
  245. stderrBuilder := strings.Builder{}
  246. if err := cmd.Run(ctx, &gitcmd.RunOpts{
  247. Timeout: timeout,
  248. Dir: repoPath,
  249. Env: envs,
  250. Stdout: &stdoutBuilder,
  251. Stderr: &stderrBuilder,
  252. }); err != nil {
  253. stdout := stdoutBuilder.String()
  254. stderr := stderrBuilder.String()
  255. // sanitize the output, since it may contain the remote address, which may contain a password
  256. stderrMessage := util.SanitizeCredentialURLs(stderr)
  257. stdoutMessage := util.SanitizeCredentialURLs(stdout)
  258. // Now check if the error is a resolve reference due to broken reference
  259. if checkRecoverableSyncError(stderr) {
  260. log.Warn("SyncMirrors [repo: %-v]: failed to update mirror repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
  261. err = nil
  262. // Attempt prune
  263. pruneErr := pruneBrokenReferences(ctx, m, timeout, &stdoutBuilder, &stderrBuilder, false)
  264. if pruneErr == nil {
  265. // Successful prune - reattempt mirror
  266. stderrBuilder.Reset()
  267. stdoutBuilder.Reset()
  268. if err = cmd.Run(ctx, &gitcmd.RunOpts{
  269. Timeout: timeout,
  270. Dir: repoPath,
  271. Stdout: &stdoutBuilder,
  272. Stderr: &stderrBuilder,
  273. }); err != nil {
  274. stdout := stdoutBuilder.String()
  275. stderr := stderrBuilder.String()
  276. // sanitize the output, since it may contain the remote address, which may
  277. // contain a password
  278. stderrMessage = util.SanitizeCredentialURLs(stderr)
  279. stdoutMessage = util.SanitizeCredentialURLs(stdout)
  280. }
  281. }
  282. }
  283. // If there is still an error (or there always was an error)
  284. if err != nil {
  285. log.Error("SyncMirrors [repo: %-v]: failed to update mirror repository:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err)
  286. desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, stderrMessage)
  287. if err = system_model.CreateRepositoryNotice(desc); err != nil {
  288. log.Error("CreateRepositoryNotice: %v", err)
  289. }
  290. return nil, false
  291. }
  292. }
  293. output := stderrBuilder.String()
  294. if err := git.WriteCommitGraph(ctx, repoPath); err != nil {
  295. log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err)
  296. }
  297. gitRepo, err := gitrepo.OpenRepository(ctx, m.Repo)
  298. if err != nil {
  299. log.Error("SyncMirrors [repo: %-v]: failed to OpenRepository: %v", m.Repo, err)
  300. return nil, false
  301. }
  302. if m.LFS && setting.LFS.StartServer {
  303. log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
  304. endpoint := lfs.DetermineEndpoint(remoteURL.String(), m.LFSEndpoint)
  305. lfsClient := lfs.NewClient(endpoint, nil)
  306. if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsClient); err != nil {
  307. log.Error("SyncMirrors [repo: %-v]: failed to synchronize LFS objects for repository: %v", m.Repo.FullName(), err)
  308. }
  309. }
  310. log.Trace("SyncMirrors [repo: %-v]: syncing branches...", m.Repo)
  311. if _, err = repo_module.SyncRepoBranchesWithRepo(ctx, m.Repo, gitRepo, 0); err != nil {
  312. log.Error("SyncMirrors [repo: %-v]: failed to synchronize branches: %v", m.Repo, err)
  313. }
  314. log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo)
  315. if err = repo_module.SyncReleasesWithTags(ctx, m.Repo, gitRepo); err != nil {
  316. log.Error("SyncMirrors [repo: %-v]: failed to synchronize tags to releases: %v", m.Repo, err)
  317. }
  318. gitRepo.Close()
  319. log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
  320. if err := repo_module.UpdateRepoSize(ctx, m.Repo); err != nil {
  321. log.Error("SyncMirrors [repo: %-v]: failed to update size for mirror repository: %v", m.Repo.FullName(), err)
  322. }
  323. if repo_service.HasWiki(ctx, m.Repo) {
  324. log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo)
  325. stderrBuilder.Reset()
  326. stdoutBuilder.Reset()
  327. if err := gitrepo.GitRemoteUpdatePrune(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName(),
  328. timeout, &stdoutBuilder, &stderrBuilder); err != nil {
  329. stdout := stdoutBuilder.String()
  330. stderr := stderrBuilder.String()
  331. // sanitize the output, since it may contain the remote address, which may contain a password
  332. stderrMessage := util.SanitizeCredentialURLs(stderr)
  333. stdoutMessage := util.SanitizeCredentialURLs(stdout)
  334. // Now check if the error is a resolve reference due to broken reference
  335. if checkRecoverableSyncError(stderrMessage) {
  336. log.Warn("SyncMirrors [repo: %-v Wiki]: failed to update mirror wiki repository due to broken references:\nStdout: %s\nStderr: %s\nErr: %v\nAttempting Prune", m.Repo, stdoutMessage, stderrMessage, err)
  337. err = nil
  338. // Attempt prune
  339. pruneErr := pruneBrokenReferences(ctx, m, timeout, &stdoutBuilder, &stderrBuilder, true)
  340. if pruneErr == nil {
  341. // Successful prune - reattempt mirror
  342. stderrBuilder.Reset()
  343. stdoutBuilder.Reset()
  344. if err = gitrepo.GitRemoteUpdatePrune(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName(),
  345. timeout, &stdoutBuilder, &stderrBuilder); err != nil {
  346. stdout := stdoutBuilder.String()
  347. stderr := stderrBuilder.String()
  348. stderrMessage = util.SanitizeCredentialURLs(stderr)
  349. stdoutMessage = util.SanitizeCredentialURLs(stdout)
  350. }
  351. }
  352. }
  353. // If there is still an error (or there always was an error)
  354. if err != nil {
  355. log.Error("SyncMirrors [repo: %-v Wiki]: failed to update mirror repository wiki:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err)
  356. desc := fmt.Sprintf("Failed to update mirror repository wiki '%s': %s", wikiPath, stderrMessage)
  357. if err = system_model.CreateRepositoryNotice(desc); err != nil {
  358. log.Error("CreateRepositoryNotice: %v", err)
  359. }
  360. return nil, false
  361. }
  362. if err := git.WriteCommitGraph(ctx, wikiPath); err != nil {
  363. log.Error("SyncMirrors [repo: %-v]: %v", m.Repo, err)
  364. }
  365. }
  366. log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo)
  367. }
  368. log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo)
  369. branches, _, err := gitrepo.GetBranchesByPath(ctx, m.Repo, 0, 0)
  370. if err != nil {
  371. log.Error("SyncMirrors [repo: %-v]: failed to GetBranches: %v", m.Repo, err)
  372. return nil, false
  373. }
  374. for _, branch := range branches {
  375. cache.Remove(m.Repo.GetCommitsCountCacheKey(branch, true))
  376. }
  377. m.UpdatedUnix = timeutil.TimeStampNow()
  378. return parseRemoteUpdateOutput(output, m.GetRemoteName()), true
  379. }
  380. func getRepoPullMirrorLockKey(repoID int64) string {
  381. return fmt.Sprintf("repo_pull_mirror_%d", repoID)
  382. }
  383. // SyncPullMirror starts the sync of the pull mirror and schedules the next run.
  384. func SyncPullMirror(ctx context.Context, repoID int64) bool {
  385. log.Trace("SyncMirrors [repo_id: %v]", repoID)
  386. defer func() {
  387. err := recover()
  388. if err == nil {
  389. return
  390. }
  391. // There was a panic whilst syncMirrors...
  392. log.Error("PANIC whilst SyncMirrors[repo_id: %d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2))
  393. }()
  394. releaser, err := globallock.Lock(ctx, getRepoPullMirrorLockKey(repoID))
  395. if err != nil {
  396. log.Error("globallock.Lock(): %v", err)
  397. return false
  398. }
  399. defer releaser()
  400. m, err := repo_model.GetMirrorByRepoID(ctx, repoID)
  401. if err != nil {
  402. log.Error("SyncMirrors [repo_id: %v]: unable to GetMirrorByRepoID: %v", repoID, err)
  403. return false
  404. }
  405. _ = m.GetRepository(ctx) // force load repository of mirror
  406. ctx, _, finished := process.GetManager().AddContext(ctx, fmt.Sprintf("Syncing Mirror %s/%s", m.Repo.OwnerName, m.Repo.Name))
  407. defer finished()
  408. log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo)
  409. results, ok := runSync(ctx, m)
  410. if !ok {
  411. if err = repo_model.TouchMirror(ctx, m); err != nil {
  412. log.Error("SyncMirrors [repo: %-v]: failed to TouchMirror: %v", m.Repo, err)
  413. }
  414. return false
  415. }
  416. log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo)
  417. m.ScheduleNextUpdate()
  418. if err = repo_model.UpdateMirror(ctx, m); err != nil {
  419. log.Error("SyncMirrors [repo: %-v]: failed to UpdateMirror with next update date: %v", m.Repo, err)
  420. return false
  421. }
  422. gitRepo, err := gitrepo.OpenRepository(ctx, m.Repo)
  423. if err != nil {
  424. log.Error("SyncMirrors [repo: %-v]: unable to OpenRepository: %v", m.Repo, err)
  425. return false
  426. }
  427. defer gitRepo.Close()
  428. log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results))
  429. if len(results) > 0 {
  430. if ok := checkAndUpdateEmptyRepository(ctx, m, results); !ok {
  431. log.Error("SyncMirrors [repo: %-v]: checkAndUpdateEmptyRepository: %v", m.Repo, err)
  432. return false
  433. }
  434. }
  435. for _, result := range results {
  436. // Discard GitHub pull requests, i.e. refs/pull/*
  437. if result.refName.IsPull() {
  438. continue
  439. }
  440. // Create reference
  441. if result.oldCommitID == gitShortEmptySha {
  442. commitID, err := gitRepo.GetRefCommitID(result.refName.String())
  443. if err != nil {
  444. log.Error("SyncMirrors [repo: %-v]: unable to GetRefCommitID [ref_name: %s]: %v", m.Repo, result.refName, err)
  445. continue
  446. }
  447. objectFormat := git.ObjectFormatFromName(m.Repo.ObjectFormatName)
  448. notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
  449. RefFullName: result.refName,
  450. OldCommitID: objectFormat.EmptyObjectID().String(),
  451. NewCommitID: commitID,
  452. }, repo_module.NewPushCommits())
  453. notify_service.SyncCreateRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName, commitID)
  454. continue
  455. }
  456. // Delete reference
  457. if result.newCommitID == gitShortEmptySha {
  458. notify_service.SyncDeleteRef(ctx, m.Repo.MustOwner(ctx), m.Repo, result.refName)
  459. continue
  460. }
  461. // Push commits
  462. oldCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.oldCommitID)
  463. if err != nil {
  464. log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID[%s]: %v", m.Repo, result.oldCommitID, err)
  465. continue
  466. }
  467. newCommitID, err := git.GetFullCommitID(gitRepo.Ctx, gitRepo.Path, result.newCommitID)
  468. if err != nil {
  469. log.Error("SyncMirrors [repo: %-v]: unable to get GetFullCommitID [%s]: %v", m.Repo, result.newCommitID, err)
  470. continue
  471. }
  472. commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID)
  473. if err != nil {
  474. log.Error("SyncMirrors [repo: %-v]: unable to get CommitsBetweenIDs [new_commit_id: %s, old_commit_id: %s]: %v", m.Repo, newCommitID, oldCommitID, err)
  475. continue
  476. }
  477. theCommits := repo_module.GitToPushCommits(commits)
  478. if len(theCommits.Commits) > setting.UI.FeedMaxCommitNum {
  479. theCommits.Commits = theCommits.Commits[:setting.UI.FeedMaxCommitNum]
  480. }
  481. newCommit, err := gitRepo.GetCommit(newCommitID)
  482. if err != nil {
  483. log.Error("SyncMirrors [repo: %-v]: unable to get commit %s: %v", m.Repo, newCommitID, err)
  484. continue
  485. }
  486. theCommits.HeadCommit = repo_module.CommitToPushCommit(newCommit)
  487. theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID)
  488. notify_service.SyncPushCommits(ctx, m.Repo.MustOwner(ctx), m.Repo, &repo_module.PushUpdateOptions{
  489. RefFullName: result.refName,
  490. OldCommitID: oldCommitID,
  491. NewCommitID: newCommitID,
  492. }, theCommits)
  493. }
  494. log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo)
  495. isEmpty, err := gitRepo.IsEmpty()
  496. if err != nil {
  497. log.Error("SyncMirrors [repo: %-v]: unable to check empty git repo: %v", m.Repo, err)
  498. return false
  499. }
  500. if !isEmpty {
  501. // Get latest commit date and update to current repository updated time
  502. commitDate, err := git.GetLatestCommitTime(ctx, m.Repo.RepoPath())
  503. if err != nil {
  504. log.Error("SyncMirrors [repo: %-v]: unable to GetLatestCommitDate: %v", m.Repo, err)
  505. return false
  506. }
  507. if err = repo_model.UpdateRepositoryUpdatedTime(ctx, m.RepoID, commitDate); err != nil {
  508. log.Error("SyncMirrors [repo: %-v]: unable to update repository 'updated_unix': %v", m.Repo, err)
  509. return false
  510. }
  511. }
  512. // Update License
  513. if err = repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
  514. RepoID: m.Repo.ID,
  515. }); err != nil {
  516. log.Error("SyncMirrors [repo: %-v]: unable to add repo to license updater queue: %v", m.Repo, err)
  517. return false
  518. }
  519. log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo)
  520. return true
  521. }
  522. func checkAndUpdateEmptyRepository(ctx context.Context, m *repo_model.Mirror, results []*mirrorSyncResult) bool {
  523. if !m.Repo.IsEmpty {
  524. return true
  525. }
  526. hasDefault := false
  527. hasMaster := false
  528. hasMain := false
  529. defaultBranchName := m.Repo.DefaultBranch
  530. if len(defaultBranchName) == 0 {
  531. defaultBranchName = setting.Repository.DefaultBranch
  532. }
  533. firstName := ""
  534. for _, result := range results {
  535. if !result.refName.IsBranch() {
  536. continue
  537. }
  538. name := result.refName.BranchName()
  539. if len(firstName) == 0 {
  540. firstName = name
  541. }
  542. hasDefault = hasDefault || name == defaultBranchName
  543. hasMaster = hasMaster || name == "master"
  544. hasMain = hasMain || name == "main"
  545. }
  546. if len(firstName) > 0 {
  547. if hasDefault {
  548. m.Repo.DefaultBranch = defaultBranchName
  549. } else if hasMaster {
  550. m.Repo.DefaultBranch = "master"
  551. } else if hasMain {
  552. m.Repo.DefaultBranch = "main"
  553. } else {
  554. m.Repo.DefaultBranch = firstName
  555. }
  556. // Update the git repository default branch
  557. if err := gitrepo.SetDefaultBranch(ctx, m.Repo, m.Repo.DefaultBranch); err != nil {
  558. log.Error("Failed to update default branch of underlying git repository %-v. Error: %v", m.Repo, err)
  559. return false
  560. }
  561. m.Repo.IsEmpty = false
  562. // Update the is empty and default_branch columns
  563. if err := repo_model.UpdateRepositoryColsWithAutoTime(ctx, m.Repo, "default_branch", "is_empty"); err != nil {
  564. log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err)
  565. desc := fmt.Sprintf("Failed to update default branch of repository '%s': %v", m.Repo.RepoPath(), err)
  566. if err = system_model.CreateRepositoryNotice(desc); err != nil {
  567. log.Error("CreateRepositoryNotice: %v", err)
  568. }
  569. return false
  570. }
  571. }
  572. return true
  573. }