From 90492d20f4e969022a457f5de26f09c9b969facb Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 1 Mar 2022 23:37:53 +0100 Subject: [PATCH 01/11] Improve dashboard's repo list performance - Avoid a lot of database lookups for all the repo's, by adding a undocumented "minimal" mode for this specific task, which returns the data that's only needed by this list which doesn't require any database lookups. - Makes fetching these list faster. - Less CPU overhead when a user visits home page. --- routers/api/v1/repo/repo.go | 32 ++++++++++++++++------ web_src/js/components/DashboardRepoList.js | 18 ++++-------- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 7a7fe218e820f..adfce94421061 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -215,6 +215,10 @@ func Search(ctx *context.APIContext) { return } + // Undocumented mode for internal usages, only return + // information that's useful for the dashboard's repo list. + minimalMode := ctx.FormBool("minimal") + results := make([]*api.Repository, len(repos)) for i, repo := range repos { if err = repo.GetOwner(db.DefaultContext); err != nil { @@ -224,16 +228,28 @@ func Search(ctx *context.APIContext) { }) return } - accessMode, err := models.AccessLevel(ctx.User, repo) - if err != nil { - ctx.JSON(http.StatusInternalServerError, api.SearchError{ - OK: false, - Error: err.Error(), - }) + if minimalMode { + results[i] = &api.Repository{ + ID: repo.ID, + FullName: repo.FullName(), + Private: repo.IsPrivate, + Template: repo.IsTemplate, + Mirror: repo.IsMirror, + Stars: repo.NumStars, + HTMLURL: repo.HTMLURL(), + Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate, + } + } else { + accessMode, err := models.AccessLevel(ctx.User, repo) + if err != nil { + ctx.JSON(http.StatusInternalServerError, api.SearchError{ + OK: false, + Error: err.Error(), + }) + } + results[i] = convert.ToRepo(repo, accessMode) } - results[i] = convert.ToRepo(repo, accessMode) } - ctx.SetLinkHeader(int(count), opts.PageSize) ctx.SetTotalCountHeader(count) ctx.JSON(http.StatusOK, api.SearchResults{ diff --git a/web_src/js/components/DashboardRepoList.js b/web_src/js/components/DashboardRepoList.js index 8d8f186cc632f..c33dd7ca36d33 100644 --- a/web_src/js/components/DashboardRepoList.js +++ b/web_src/js/components/DashboardRepoList.js @@ -124,7 +124,7 @@ function initVueComponents() { return this.repos.length > 0 && this.repos.length < this.counts[`${this.reposFilter}:${this.archivedFilter}:${this.privateFilter}`]; }, searchURL() { - return `${this.subUrl}/api/v1/repos/search?sort=updated&order=desc&uid=${this.uid}&team_id=${this.teamId}&q=${this.searchQuery + return `${this.subUrl}/api/v1/repos/search?sort=updated&order=desc&minimal=1&uid=${this.uid}&team_id=${this.teamId}&q=${this.searchQuery }&page=${this.page}&limit=${this.searchLimit}&mode=${this.repoTypes[this.reposFilter].searchMode }${this.reposFilter !== 'all' ? '&exclusive=1' : '' }${this.archivedFilter === 'archived' ? '&archived=true' : ''}${this.archivedFilter === 'unarchived' ? '&archived=false' : '' @@ -301,21 +301,15 @@ function initVueComponents() { searchRepos() { this.isLoading = true; - if (!this.reposTotalCount) { - const totalCountSearchURL = `${this.subUrl}/api/v1/repos/search?sort=updated&order=desc&uid=${this.uid}&team_id=${this.teamId}&q=&page=1&mode=`; - $.getJSON(totalCountSearchURL, (_result, _textStatus, request) => { - this.reposTotalCount = request.getResponseHeader('X-Total-Count'); - }); - } - const searchedMode = this.repoTypes[this.reposFilter].searchMode; const searchedURL = this.searchURL; const searchedQuery = this.searchQuery; - $.getJSON(searchedURL, (result, _textStatus, request) => { + fetch(searchedURL).then(async (response) => { + const json = await response.json(); if (searchedURL === this.searchURL) { - this.repos = result.data; - const count = request.getResponseHeader('X-Total-Count'); + this.repos = json.data; + const count = response.headers.get('X-Total-Count'); if (searchedQuery === '' && searchedMode === '' && this.archivedFilter === 'both') { this.reposTotalCount = count; } @@ -323,7 +317,7 @@ function initVueComponents() { this.finalPage = Math.ceil(count / this.searchLimit); this.updateHistory(); } - }).always(() => { + }).finally(() => { if (searchedURL === this.searchURL) { this.isLoading = false; } From 22acfd62f9ebc10ecbffb01a46504c86d9dcd4ab Mon Sep 17 00:00:00 2001 From: Gusted Date: Wed, 2 Mar 2022 12:10:05 +0100 Subject: [PATCH 02/11] Refactor javascript code + fix Fork icon - Use async in the function so we can use `await`. - Remove `archivedFilter` check for count, as it doesn't make sense to show the count of repos when you can't even see them(as they are filited away). --- routers/api/v1/repo/repo.go | 1 + web_src/js/components/DashboardRepoList.js | 36 +++++++++++++--------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index adfce94421061..6f98f40163a2b 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -232,6 +232,7 @@ func Search(ctx *context.APIContext) { results[i] = &api.Repository{ ID: repo.ID, FullName: repo.FullName(), + Fork: repo.IsFork, Private: repo.IsPrivate, Template: repo.IsTemplate, Mirror: repo.IsMirror, diff --git a/web_src/js/components/DashboardRepoList.js b/web_src/js/components/DashboardRepoList.js index c33dd7ca36d33..875281fc4c1a9 100644 --- a/web_src/js/components/DashboardRepoList.js +++ b/web_src/js/components/DashboardRepoList.js @@ -298,30 +298,36 @@ function initVueComponents() { this.searchRepos(); }, - searchRepos() { + async searchRepos() { this.isLoading = true; const searchedMode = this.repoTypes[this.reposFilter].searchMode; const searchedURL = this.searchURL; const searchedQuery = this.searchQuery; - fetch(searchedURL).then(async (response) => { - const json = await response.json(); - if (searchedURL === this.searchURL) { - this.repos = json.data; - const count = response.headers.get('X-Total-Count'); - if (searchedQuery === '' && searchedMode === '' && this.archivedFilter === 'both') { - this.reposTotalCount = count; - } - Vue.set(this.counts, `${this.reposFilter}:${this.archivedFilter}:${this.privateFilter}`, count); - this.finalPage = Math.ceil(count / this.searchLimit); - this.updateHistory(); - } - }).finally(() => { + let json = {}; + let response; + try { + response = await fetch(searchedURL); + json = await response.json(); + } catch { if (searchedURL === this.searchURL) { this.isLoading = false; } - }); + return; + } + + if (searchedURL === this.searchURL) { + this.repos = json.data; + const count = response.headers.get('X-Total-Count'); + if (searchedQuery === '' && searchedMode === '') { + this.reposTotalCount = count; + } + Vue.set(this.counts, `${this.reposFilter}:${this.archivedFilter}:${this.privateFilter}`, count); + this.finalPage = Math.ceil(count / this.searchLimit); + this.updateHistory(); + this.isLoading = false; + } }, repoIcon(repo) { From c6706e668412b2716eddd8dddfd71470bca3416e Mon Sep 17 00:00:00 2001 From: Gusted Date: Wed, 2 Mar 2022 12:54:53 +0100 Subject: [PATCH 03/11] Add `count_only` --- routers/api/v1/repo/repo.go | 7 +++++++ web_src/js/components/DashboardRepoList.js | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 6f98f40163a2b..c6578a04d562b 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -215,6 +215,13 @@ func Search(ctx *context.APIContext) { return } + // Undocumented mode for internal usages, don't send + // any data only the count. + if ctx.FormBool("count_only") { + ctx.SetTotalCountHeader(count) + return + } + // Undocumented mode for internal usages, only return // information that's useful for the dashboard's repo list. minimalMode := ctx.FormBool("minimal") diff --git a/web_src/js/components/DashboardRepoList.js b/web_src/js/components/DashboardRepoList.js index 875281fc4c1a9..362e4d542c603 100644 --- a/web_src/js/components/DashboardRepoList.js +++ b/web_src/js/components/DashboardRepoList.js @@ -308,6 +308,12 @@ function initVueComponents() { let json = {}; let response; try { + if (!this.reposTotalCount) { + const totalCountSearchURL = `${this.subUrl}/api/v1/repos/search?sort=updated&count_only=1&order=desc&uid=${this.uid}&team_id=${this.teamId}&q=&page=1&mode=`; + const response = await fetch(totalCountSearchURL); + this.reposTotalCount = response.headers.get('X-Total-Count'); + } + response = await fetch(searchedURL); json = await response.json(); } catch { @@ -320,7 +326,7 @@ function initVueComponents() { if (searchedURL === this.searchURL) { this.repos = json.data; const count = response.headers.get('X-Total-Count'); - if (searchedQuery === '' && searchedMode === '') { + if (searchedQuery === '' && searchedMode === '' && this.archivedFilter === 'both') { this.reposTotalCount = count; } Vue.set(this.counts, `${this.reposFilter}:${this.archivedFilter}:${this.privateFilter}`, count); From 53199d4a0cdb61c759f1922cc1c01646aa078359 Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 12 Apr 2022 17:25:20 +0200 Subject: [PATCH 04/11] Remove uncessary code --- routers/api/v1/repo/repo.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 0974e1880716b..d9500db3867a1 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -199,13 +199,6 @@ func Search(ctx *context.APIContext) { return } - // Undocumented mode for internal usages, don't send - // any data only the count. - if ctx.FormBool("count_only") { - ctx.SetTotalCountHeader(count) - return - } - results := make([]*api.Repository, len(repos)) for i, repo := range repos { if err = repo.GetOwner(ctx); err != nil { From 47926f59077b1263ad9bc2046edb6d3016b6cc4e Mon Sep 17 00:00:00 2001 From: Gusted Date: Thu, 14 Apr 2022 16:37:45 +0000 Subject: [PATCH 05/11] Improve comment Co-authored-by: delvh --- routers/web/repo/repo.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 1222418faf01d..f45234b799459 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -590,15 +590,13 @@ func SearchRepo(ctx *context.Context) { return } - // Undocumented mode for internal usages, don't send - // any data only the count. + // To improve performance when only the count is requested if ctx.FormBool("count_only") { ctx.SetTotalCountHeader(count) return } - // Undocumented mode for internal usages, only return - // information that's useful for the dashboard's repo list. + // To query only what's necessary for the dashboard repo list minimalMode := ctx.FormBool("minimal") results := make([]*api.Repository, len(repos)) From 1b42dd23ab4296a661ee1eeb6cf2f5bb4486d507 Mon Sep 17 00:00:00 2001 From: Gusted Date: Thu, 14 Apr 2022 16:40:20 +0000 Subject: [PATCH 06/11] Update web_src/js/components/DashboardRepoList.js Co-authored-by: delvh --- web_src/js/components/DashboardRepoList.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/components/DashboardRepoList.js b/web_src/js/components/DashboardRepoList.js index a9d1c6576cafa..fa84cee2b3375 100644 --- a/web_src/js/components/DashboardRepoList.js +++ b/web_src/js/components/DashboardRepoList.js @@ -310,7 +310,7 @@ function initVueComponents() { try { if (!this.reposTotalCount) { const totalCountSearchURL = `${this.subUrl}/repo/search?sort=updated&count_only=1&order=desc&uid=${this.uid}&team_id=${this.teamId}&q=&page=1&mode=`; - const response = await fetch(totalCountSearchURL); + response = await fetch(totalCountSearchURL); this.reposTotalCount = response.headers.get('X-Total-Count'); } From 1444f0619af9a9e3005216050093c83ae21b535a Mon Sep 17 00:00:00 2001 From: Gusted Date: Fri, 15 Apr 2022 10:47:20 +0200 Subject: [PATCH 07/11] Update web_src/js/components/DashboardRepoList.js Co-authored-by: delvh --- web_src/js/components/DashboardRepoList.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/components/DashboardRepoList.js b/web_src/js/components/DashboardRepoList.js index fa84cee2b3375..ab6dad2b1afc9 100644 --- a/web_src/js/components/DashboardRepoList.js +++ b/web_src/js/components/DashboardRepoList.js @@ -309,7 +309,7 @@ function initVueComponents() { let response; try { if (!this.reposTotalCount) { - const totalCountSearchURL = `${this.subUrl}/repo/search?sort=updated&count_only=1&order=desc&uid=${this.uid}&team_id=${this.teamId}&q=&page=1&mode=`; + const totalCountSearchURL = `${this.subUrl}/repo/search?count_only=1&uid=${this.uid}&team_id=${this.teamId}&q=&page=1&mode=`; response = await fetch(totalCountSearchURL); this.reposTotalCount = response.headers.get('X-Total-Count'); } From 19f811f73b79ebf99b9272c51cd6e7166c87cadd Mon Sep 17 00:00:00 2001 From: Gusted Date: Fri, 15 Apr 2022 19:27:25 +0200 Subject: [PATCH 08/11] By default apply minimal mode --- routers/web/repo/repo.go | 41 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index f45234b799459..6bb354b43ca18 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -596,39 +596,18 @@ func SearchRepo(ctx *context.Context) { return } - // To query only what's necessary for the dashboard repo list - minimalMode := ctx.FormBool("minimal") - results := make([]*api.Repository, len(repos)) for i, repo := range repos { - if err = repo.GetOwner(ctx); err != nil { - ctx.JSON(http.StatusInternalServerError, api.SearchError{ - OK: false, - Error: err.Error(), - }) - return - } - if minimalMode { - results[i] = &api.Repository{ - ID: repo.ID, - FullName: repo.FullName(), - Fork: repo.IsFork, - Private: repo.IsPrivate, - Template: repo.IsTemplate, - Mirror: repo.IsMirror, - Stars: repo.NumStars, - HTMLURL: repo.HTMLURL(), - Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate, - } - } else { - accessMode, err := models.AccessLevel(ctx.Doer, repo) - if err != nil { - ctx.JSON(http.StatusInternalServerError, api.SearchError{ - OK: false, - Error: err.Error(), - }) - } - results[i] = convert.ToRepo(repo, accessMode) + results[i] = &api.Repository{ + ID: repo.ID, + FullName: repo.FullName(), + Fork: repo.IsFork, + Private: repo.IsPrivate, + Template: repo.IsTemplate, + Mirror: repo.IsMirror, + Stars: repo.NumStars, + HTMLURL: repo.HTMLURL(), + Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate, } } From 3b67a8c817a6c05db28f9a00fdf1e07b89e6ae41 Mon Sep 17 00:00:00 2001 From: Gusted Date: Fri, 15 Apr 2022 19:28:29 +0200 Subject: [PATCH 09/11] Remove `minimal` paramater --- web_src/js/components/DashboardRepoList.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/components/DashboardRepoList.js b/web_src/js/components/DashboardRepoList.js index ab6dad2b1afc9..98bbf0f3a3fb9 100644 --- a/web_src/js/components/DashboardRepoList.js +++ b/web_src/js/components/DashboardRepoList.js @@ -124,7 +124,7 @@ function initVueComponents() { return this.repos.length > 0 && this.repos.length < this.counts[`${this.reposFilter}:${this.archivedFilter}:${this.privateFilter}`]; }, searchURL() { - return `${this.subUrl}/repo/search?sort=updated&order=desc&minimal=1&uid=${this.uid}&team_id=${this.teamId}&q=${this.searchQuery + return `${this.subUrl}/repo/search?sort=updated&order=desc&uid=${this.uid}&team_id=${this.teamId}&q=${this.searchQuery }&page=${this.page}&limit=${this.searchLimit}&mode=${this.repoTypes[this.reposFilter].searchMode }${this.reposFilter !== 'all' ? '&exclusive=1' : '' }${this.archivedFilter === 'archived' ? '&archived=true' : ''}${this.archivedFilter === 'unarchived' ? '&archived=false' : '' From 3b91ad4c406b49c445c030005a1d4542e4ccf849 Mon Sep 17 00:00:00 2001 From: Gusted Date: Mon, 25 Apr 2022 21:35:56 +0200 Subject: [PATCH 10/11] Refactor count header --- routers/web/repo/repo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 6bb354b43ca18..199651b2f125a 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -590,9 +590,10 @@ func SearchRepo(ctx *context.Context) { return } + ctx.SetTotalCountHeader(count) + // To improve performance when only the count is requested if ctx.FormBool("count_only") { - ctx.SetTotalCountHeader(count) return } @@ -611,7 +612,6 @@ func SearchRepo(ctx *context.Context) { } } - ctx.SetTotalCountHeader(count) ctx.JSON(http.StatusOK, api.SearchResults{ OK: true, Data: results, From fab1d89e898cd9d066f90a86dfbbd54e961c2c56 Mon Sep 17 00:00:00 2001 From: Gusted Date: Tue, 26 Apr 2022 13:02:39 +0000 Subject: [PATCH 11/11] Simplify init Co-authored-by: wxiaoguang --- web_src/js/components/DashboardRepoList.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web_src/js/components/DashboardRepoList.js b/web_src/js/components/DashboardRepoList.js index 98bbf0f3a3fb9..36caaf2f5bcfc 100644 --- a/web_src/js/components/DashboardRepoList.js +++ b/web_src/js/components/DashboardRepoList.js @@ -305,8 +305,7 @@ function initVueComponents() { const searchedURL = this.searchURL; const searchedQuery = this.searchQuery; - let json = {}; - let response; + let response, json; try { if (!this.reposTotalCount) { const totalCountSearchURL = `${this.subUrl}/repo/search?count_only=1&uid=${this.uid}&team_id=${this.teamId}&q=&page=1&mode=`;