From 8f2f74c77c2fcd683396ff6f2e9a39cf94868e73 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:41:38 +0200 Subject: [PATCH 01/18] new http score --- scripts/sort-libraries.ts | 234 -------------------------------------- 1 file changed, 234 deletions(-) delete mode 100644 scripts/sort-libraries.ts diff --git a/scripts/sort-libraries.ts b/scripts/sort-libraries.ts deleted file mode 100644 index bf053346c1..0000000000 --- a/scripts/sort-libraries.ts +++ /dev/null @@ -1,234 +0,0 @@ -const numbro = require("numbro") -const timeago = require("timeago.js") - -const getGitHubStats = async githubRepo => { - const [owner, repoName] = githubRepo.split("/") - const accessToken = process.env.GITHUB_ACCESS_TOKEN - if (!accessToken) { - return { - accessToken: false, - } - } - const query = /* GraphQL */ ` - fragment defaultBranchRefFragment on Ref { - target { - ... on Commit { - history(since: $since) { - edges { - node { - author { - name - } - pushedDate - } - } - } - } - } - } - query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { - repositoryOwner(login: $owner) { - repository(name: $repoName) { - defaultBranchRef { - ...defaultBranchRefFragment - } - stargazers { - totalCount - } - updatedAt - forkCount - pullRequests { - totalCount - } - description - licenseInfo { - name - } - releases(first: 1) { - nodes { - publishedAt - } - } - tags: refs( - refPrefix: "refs/tags/" - first: 1 - orderBy: { field: TAG_COMMIT_DATE, direction: DESC } - ) { - nodes { - name - target { - ... on Tag { - target { - ... on Commit { - pushedDate - } - } - } - } - } - } - } - } - } - ` - const lastMonth = new Date() - lastMonth.setMonth(lastMonth.getMonth() - 3) - const response = await fetch("https://api.github.com/graphql", { - method: "POST", - body: JSON.stringify({ - query, - variables: { owner, repoName, since: lastMonth }, - }), - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }) - const responseJson = await response.json() - if (responseJson && responseJson.errors) { - throw JSON.stringify(responseJson.errors) - } - if (!responseJson || !responseJson.data) { - throw `GitHub returned empty response for ${owner}/${repoName}` - } - const { repositoryOwner } = responseJson.data - if (!repositoryOwner) { - throw `No GitHub user found for ${owner}/${repoName}` - } - const { repository: repo } = repositoryOwner - if (!repo) { - throw `No GitHub repo found ${owner}/${repoName}` - } - const stars = repo.stargazers.totalCount - const commitHistory = repo.defaultBranchRef.target.history.edges - - let hasCommitsInLast3Months = false - commitHistory.forEach(commit => { - if (!commit.node.author.name.match(/bot/i)) { - hasCommitsInLast3Months = true - } - }) - const formattedStars = numbro(stars).format({ - average: true, - }) - - const releases: any = [] - if ( - repo.tags && - repo.tags.nodes && - repo.tags.nodes.length && - repo.tags.nodes[0].target.target && - repo.tags.nodes[0].target.target.pushedDate - ) { - releases.push(repo.tags.nodes[0].target.target.pushedDate) - } - if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { - releases.push(repo.releases.nodes[0].publishedAt) - } - if (owner.includes("graphql")) { - console.log({ releases, repoName }) - } - - const lastRelease = releases.filter(Boolean).sort().reverse()[0] - - return { - hasCommitsInLast3Months, - stars, - formattedStars, - license: repo.licenseInfo && repo.licenseInfo.name, - lastRelease, - formattedLastRelease: lastRelease && timeago.format(lastRelease), - } -} - -async function getNpmStats(packageName: string): Promise { - const response = await fetch( - `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( - packageName - )}` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - if (!downloadCount) { - console.debug( - `getNpmStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} - -async function getGemStats(packageName: string): Promise { - const response = await fetch( - `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - console.debug(`getGemStats: ${downloadCount} for ${packageName}`) - if (!downloadCount) { - console.debug( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} - -export async function sortLibs(libs: any) { - { - let totalStars = 0 - const libsWithScores = await Promise.all( - libs.map(async lib => { - console.log(`Fetching stats for ${lib.name}`) - console.log(`Fetching gem for ${lib.gem}`) - console.log(`Fetching github for ${lib.github}`) - const [npmStats = {}, gemStars = {}, githubStats = {}] = - await Promise.all([ - lib.npm && (await getNpmStats(lib.npm)), - lib.gem && (await getGemStats(lib.gem)), - lib.github && (await getGitHubStats(lib.github)), - ]) - const result = { - ...lib, - ...npmStats, - ...gemStars, - ...githubStats, - } - totalStars += result.stars || 0 - return result - }) - ) - const sortedLibs = libsWithScores.sort((a, b) => { - let aScore = 0, - bScore = 0 - if ("downloadCount" in a && "downloadCount" in b) { - if (a.downloadCount > b.downloadCount) { - aScore += 40 - } else if (b.downloadCount > a.downloadCount) { - bScore += 40 - } - } - if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { - aScore += 30 - } - if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { - bScore += 30 - } - if ("stars" in a && "stars" in b) { - if (a.stars > b.stars) { - aScore += 40 - } else if (a.stars < b.stars) { - bScore += 40 - } - } - if (bScore > aScore) { - return 1 - } - if (bScore < aScore) { - return -1 - } - return 0 - }) - return { sortedLibs, totalStars } - } -} From ee387f76573e46dc470aeab0e81a5c4f930b51ac Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:44:07 +0200 Subject: [PATCH 02/18] fix Signed-off-by: Tuval Simha --- scripts/sort-libraries/sort-libraries.ts | 234 +++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 scripts/sort-libraries/sort-libraries.ts diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts new file mode 100644 index 0000000000..bf053346c1 --- /dev/null +++ b/scripts/sort-libraries/sort-libraries.ts @@ -0,0 +1,234 @@ +const numbro = require("numbro") +const timeago = require("timeago.js") + +const getGitHubStats = async githubRepo => { + const [owner, repoName] = githubRepo.split("/") + const accessToken = process.env.GITHUB_ACCESS_TOKEN + if (!accessToken) { + return { + accessToken: false, + } + } + const query = /* GraphQL */ ` + fragment defaultBranchRefFragment on Ref { + target { + ... on Commit { + history(since: $since) { + edges { + node { + author { + name + } + pushedDate + } + } + } + } + } + } + query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { + repositoryOwner(login: $owner) { + repository(name: $repoName) { + defaultBranchRef { + ...defaultBranchRefFragment + } + stargazers { + totalCount + } + updatedAt + forkCount + pullRequests { + totalCount + } + description + licenseInfo { + name + } + releases(first: 1) { + nodes { + publishedAt + } + } + tags: refs( + refPrefix: "refs/tags/" + first: 1 + orderBy: { field: TAG_COMMIT_DATE, direction: DESC } + ) { + nodes { + name + target { + ... on Tag { + target { + ... on Commit { + pushedDate + } + } + } + } + } + } + } + } + } + ` + const lastMonth = new Date() + lastMonth.setMonth(lastMonth.getMonth() - 3) + const response = await fetch("https://api.github.com/graphql", { + method: "POST", + body: JSON.stringify({ + query, + variables: { owner, repoName, since: lastMonth }, + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }) + const responseJson = await response.json() + if (responseJson && responseJson.errors) { + throw JSON.stringify(responseJson.errors) + } + if (!responseJson || !responseJson.data) { + throw `GitHub returned empty response for ${owner}/${repoName}` + } + const { repositoryOwner } = responseJson.data + if (!repositoryOwner) { + throw `No GitHub user found for ${owner}/${repoName}` + } + const { repository: repo } = repositoryOwner + if (!repo) { + throw `No GitHub repo found ${owner}/${repoName}` + } + const stars = repo.stargazers.totalCount + const commitHistory = repo.defaultBranchRef.target.history.edges + + let hasCommitsInLast3Months = false + commitHistory.forEach(commit => { + if (!commit.node.author.name.match(/bot/i)) { + hasCommitsInLast3Months = true + } + }) + const formattedStars = numbro(stars).format({ + average: true, + }) + + const releases: any = [] + if ( + repo.tags && + repo.tags.nodes && + repo.tags.nodes.length && + repo.tags.nodes[0].target.target && + repo.tags.nodes[0].target.target.pushedDate + ) { + releases.push(repo.tags.nodes[0].target.target.pushedDate) + } + if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { + releases.push(repo.releases.nodes[0].publishedAt) + } + if (owner.includes("graphql")) { + console.log({ releases, repoName }) + } + + const lastRelease = releases.filter(Boolean).sort().reverse()[0] + + return { + hasCommitsInLast3Months, + stars, + formattedStars, + license: repo.licenseInfo && repo.licenseInfo.name, + lastRelease, + formattedLastRelease: lastRelease && timeago.format(lastRelease), + } +} + +async function getNpmStats(packageName: string): Promise { + const response = await fetch( + `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( + packageName + )}` + ) + const responseJson = await response.json() + const downloadCount = responseJson.downloads + if (!downloadCount) { + console.debug( + `getNpmStats: No download count for ${packageName}, so value is 0!` + ) + return 0 + } + return downloadCount +} + +async function getGemStats(packageName: string): Promise { + const response = await fetch( + `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` + ) + const responseJson = await response.json() + const downloadCount = responseJson.downloads + console.debug(`getGemStats: ${downloadCount} for ${packageName}`) + if (!downloadCount) { + console.debug( + `getGemStats: No download count for ${packageName}, so value is 0!` + ) + return 0 + } + return downloadCount +} + +export async function sortLibs(libs: any) { + { + let totalStars = 0 + const libsWithScores = await Promise.all( + libs.map(async lib => { + console.log(`Fetching stats for ${lib.name}`) + console.log(`Fetching gem for ${lib.gem}`) + console.log(`Fetching github for ${lib.github}`) + const [npmStats = {}, gemStars = {}, githubStats = {}] = + await Promise.all([ + lib.npm && (await getNpmStats(lib.npm)), + lib.gem && (await getGemStats(lib.gem)), + lib.github && (await getGitHubStats(lib.github)), + ]) + const result = { + ...lib, + ...npmStats, + ...gemStars, + ...githubStats, + } + totalStars += result.stars || 0 + return result + }) + ) + const sortedLibs = libsWithScores.sort((a, b) => { + let aScore = 0, + bScore = 0 + if ("downloadCount" in a && "downloadCount" in b) { + if (a.downloadCount > b.downloadCount) { + aScore += 40 + } else if (b.downloadCount > a.downloadCount) { + bScore += 40 + } + } + if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { + aScore += 30 + } + if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { + bScore += 30 + } + if ("stars" in a && "stars" in b) { + if (a.stars > b.stars) { + aScore += 40 + } else if (a.stars < b.stars) { + bScore += 40 + } + } + if (bScore > aScore) { + return 1 + } + if (bScore < aScore) { + return -1 + } + return 0 + }) + return { sortedLibs, totalStars } + } +} From 4c291dcabf43c406511aad52e094b38a73bf99c9 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:46:08 +0200 Subject: [PATCH 03/18] get stats function Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-gem-stats.ts | 33 ++++++++++++++++++++++++ scripts/sort-libraries/sort-libraries.ts | 17 ++---------- 2 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 scripts/sort-libraries/get-gem-stats.ts diff --git a/scripts/sort-libraries/get-gem-stats.ts b/scripts/sort-libraries/get-gem-stats.ts new file mode 100644 index 0000000000..d3b5b1a9b2 --- /dev/null +++ b/scripts/sort-libraries/get-gem-stats.ts @@ -0,0 +1,33 @@ +type GemStatsRespone = { + name?: string + downloads?: number + version?: string + version_created_at?: string + version_downloads?: number + platform?: string + authors?: string + info?: string + licenses?: Array +} + +export async function getGemStats( + packageName: string +): Promise { + const response = await fetch( + `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` + ) + if (!response.ok) { + console.warn( + `getGemStats: No download count for ${packageName}, so value is 0!` + ) + return { downloads: 0 } + } + const responseJson: GemStatsRespone = await response.json() + if (!responseJson) { + console.warn( + `getGemStats: No download count for ${packageName}, so value is 0!` + ) + return { downloads: 0 } + } + return responseJson +} \ No newline at end of file diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index bf053346c1..f4773b82de 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -1,3 +1,5 @@ +import { getGemStats } from "./get-gem-stats" + const numbro = require("numbro") const timeago = require("timeago.js") @@ -158,21 +160,6 @@ async function getNpmStats(packageName: string): Promise { return downloadCount } -async function getGemStats(packageName: string): Promise { - const response = await fetch( - `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - console.debug(`getGemStats: ${downloadCount} for ${packageName}`) - if (!downloadCount) { - console.debug( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} export async function sortLibs(libs: any) { { From 4fc5d9df302638f53606802a0be38dc40fe26919 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:47:42 +0200 Subject: [PATCH 04/18] getGitHubStats function Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-github-stats.ts | 142 ++++++++++++++++++++ scripts/sort-libraries/sort-libraries.ts | 143 --------------------- 2 files changed, 142 insertions(+), 143 deletions(-) create mode 100644 scripts/sort-libraries/get-github-stats.ts diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts new file mode 100644 index 0000000000..b3313a9ea3 --- /dev/null +++ b/scripts/sort-libraries/get-github-stats.ts @@ -0,0 +1,142 @@ +const numbro = require("numbro") +const timeago = require("timeago.js") + +export async function getGitHubStats(githubRepo: any) { + const [owner, repoName] = githubRepo.split("/") + const accessToken = process.env.GITHUB_ACCESS_TOKEN + if (!accessToken) { + return { + accessToken: false, + } + } + const query = /* GraphQL */ ` + fragment defaultBranchRefFragment on Ref { + target { + ... on Commit { + history(since: $since) { + edges { + node { + author { + name + } + pushedDate + } + } + } + } + } + } + query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { + repositoryOwner(login: $owner) { + repository(name: $repoName) { + defaultBranchRef { + ...defaultBranchRefFragment + } + stargazers { + totalCount + } + updatedAt + forkCount + pullRequests { + totalCount + } + description + licenseInfo { + name + } + releases(first: 1) { + nodes { + publishedAt + } + } + tags: refs( + refPrefix: "refs/tags/" + first: 1 + orderBy: { field: TAG_COMMIT_DATE, direction: DESC } + ) { + nodes { + name + target { + ... on Tag { + target { + ... on Commit { + pushedDate + } + } + } + } + } + } + } + } + } + ` + const lastMonth = new Date() + lastMonth.setMonth(lastMonth.getMonth() - 3) + const response = await fetch("https://api.github.com/graphql", { + method: "POST", + body: JSON.stringify({ + query, + variables: { owner, repoName, since: lastMonth }, + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }) + const responseJson = await response.json() + if (responseJson && responseJson.errors) { + throw JSON.stringify(responseJson.errors) + } + if (!responseJson || !responseJson.data) { + throw `GitHub returned empty response for ${owner}/${repoName}` + } + const { repositoryOwner } = responseJson.data + if (!repositoryOwner) { + throw `No GitHub user found for ${owner}/${repoName}` + } + const { repository: repo } = repositoryOwner + if (!repo) { + throw `No GitHub repo found ${owner}/${repoName}` + } + const stars = repo.stargazers.totalCount + const commitHistory = repo.defaultBranchRef.target.history.edges + + let hasCommitsInLast3Months = false + commitHistory.forEach(commit => { + if (!commit.node.author.name.match(/bot/i)) { + hasCommitsInLast3Months = true + } + }) + const formattedStars = numbro(stars).format({ + average: true, + }) + + const releases: any = [] + if ( + repo.tags && + repo.tags.nodes && + repo.tags.nodes.length && + repo.tags.nodes[0].target.target && + repo.tags.nodes[0].target.target.pushedDate + ) { + releases.push(repo.tags.nodes[0].target.target.pushedDate) + } + if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { + releases.push(repo.releases.nodes[0].publishedAt) + } + if (owner.includes("graphql")) { + console.log({ releases, repoName }) + } + + const lastRelease = releases.filter(Boolean).sort().reverse()[0] + + return { + hasCommitsInLast3Months, + stars, + formattedStars, + license: repo.licenseInfo && repo.licenseInfo.name, + lastRelease, + formattedLastRelease: lastRelease && timeago.format(lastRelease), + } +} \ No newline at end of file diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index f4773b82de..281e9897dc 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -1,148 +1,5 @@ import { getGemStats } from "./get-gem-stats" -const numbro = require("numbro") -const timeago = require("timeago.js") - -const getGitHubStats = async githubRepo => { - const [owner, repoName] = githubRepo.split("/") - const accessToken = process.env.GITHUB_ACCESS_TOKEN - if (!accessToken) { - return { - accessToken: false, - } - } - const query = /* GraphQL */ ` - fragment defaultBranchRefFragment on Ref { - target { - ... on Commit { - history(since: $since) { - edges { - node { - author { - name - } - pushedDate - } - } - } - } - } - } - query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { - repositoryOwner(login: $owner) { - repository(name: $repoName) { - defaultBranchRef { - ...defaultBranchRefFragment - } - stargazers { - totalCount - } - updatedAt - forkCount - pullRequests { - totalCount - } - description - licenseInfo { - name - } - releases(first: 1) { - nodes { - publishedAt - } - } - tags: refs( - refPrefix: "refs/tags/" - first: 1 - orderBy: { field: TAG_COMMIT_DATE, direction: DESC } - ) { - nodes { - name - target { - ... on Tag { - target { - ... on Commit { - pushedDate - } - } - } - } - } - } - } - } - } - ` - const lastMonth = new Date() - lastMonth.setMonth(lastMonth.getMonth() - 3) - const response = await fetch("https://api.github.com/graphql", { - method: "POST", - body: JSON.stringify({ - query, - variables: { owner, repoName, since: lastMonth }, - }), - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }) - const responseJson = await response.json() - if (responseJson && responseJson.errors) { - throw JSON.stringify(responseJson.errors) - } - if (!responseJson || !responseJson.data) { - throw `GitHub returned empty response for ${owner}/${repoName}` - } - const { repositoryOwner } = responseJson.data - if (!repositoryOwner) { - throw `No GitHub user found for ${owner}/${repoName}` - } - const { repository: repo } = repositoryOwner - if (!repo) { - throw `No GitHub repo found ${owner}/${repoName}` - } - const stars = repo.stargazers.totalCount - const commitHistory = repo.defaultBranchRef.target.history.edges - - let hasCommitsInLast3Months = false - commitHistory.forEach(commit => { - if (!commit.node.author.name.match(/bot/i)) { - hasCommitsInLast3Months = true - } - }) - const formattedStars = numbro(stars).format({ - average: true, - }) - - const releases: any = [] - if ( - repo.tags && - repo.tags.nodes && - repo.tags.nodes.length && - repo.tags.nodes[0].target.target && - repo.tags.nodes[0].target.target.pushedDate - ) { - releases.push(repo.tags.nodes[0].target.target.pushedDate) - } - if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { - releases.push(repo.releases.nodes[0].publishedAt) - } - if (owner.includes("graphql")) { - console.log({ releases, repoName }) - } - - const lastRelease = releases.filter(Boolean).sort().reverse()[0] - - return { - hasCommitsInLast3Months, - stars, - formattedStars, - license: repo.licenseInfo && repo.licenseInfo.name, - lastRelease, - formattedLastRelease: lastRelease && timeago.format(lastRelease), - } -} - async function getNpmStats(packageName: string): Promise { const response = await fetch( `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( From fe236ab9e9ef73a755da578c3a150c9e88a422a7 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:48:36 +0200 Subject: [PATCH 05/18] getHttpScore function Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-http-score.ts | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 scripts/sort-libraries/get-http-score.ts diff --git a/scripts/sort-libraries/get-http-score.ts b/scripts/sort-libraries/get-http-score.ts new file mode 100644 index 0000000000..31340a1d92 --- /dev/null +++ b/scripts/sort-libraries/get-http-score.ts @@ -0,0 +1,30 @@ +type HttpScoreRespone = { + total?: number + ok?: number + warn?: number + error?: number +} + +export async function getHttpScore( + packageName: string +): Promise { + const response = await fetch( + `https://raw.githubusercontent.com/graphql/graphql-http/main/implementations/${encodeURIComponent( + packageName + )}/report.json` + ) + if (!response.ok) { + console.warn( + `getHttpScore: No download count for ${packageName}, so value is 0!` + ) + return { total: 0 } + } + const responseJson: HttpScoreRespone = await response.json() + if (!responseJson) { + console.warn( + `getHttpScore: No download count for ${packageName}, so value is 0!` + ) + return { total: 0 } + } + return responseJson +} \ No newline at end of file From 204197ac8c41ff856428ef9f3d7db133a5a7e0c9 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:49:49 +0200 Subject: [PATCH 06/18] getNpmStats function Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-npm-stats.ts | 30 ++++++++++++++++++++++++ scripts/sort-libraries/sort-libraries.ts | 19 ++------------- 2 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 scripts/sort-libraries/get-npm-stats.ts diff --git a/scripts/sort-libraries/get-npm-stats.ts b/scripts/sort-libraries/get-npm-stats.ts new file mode 100644 index 0000000000..8cd33bd6c4 --- /dev/null +++ b/scripts/sort-libraries/get-npm-stats.ts @@ -0,0 +1,30 @@ +type NpmStatsResponse = { + downloads?: number + start?: string + end?: string + package?: string +} + +export async function getNpmStats( + packageName: string +): Promise { + const response = await fetch( + `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( + packageName + )}` + ) + if (!response.ok) { + console.warn( + `getGemStats: No download count for ${packageName}, so value is 0!` + ) + return { downloads: 0 } + } + const responseJson: NpmStatsResponse = await response.json() + if (!responseJson) { + console.warn( + `getGemStats: No download count for ${packageName}, so value is 0!` + ) + return { downloads: 0 } + } + return responseJson +} \ No newline at end of file diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index 281e9897dc..af8e81a238 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -1,21 +1,6 @@ import { getGemStats } from "./get-gem-stats" - -async function getNpmStats(packageName: string): Promise { - const response = await fetch( - `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( - packageName - )}` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - if (!downloadCount) { - console.debug( - `getNpmStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} +import { getGitHubStats } from "./get-github-stats" +import { getNpmStats } from "./get-npm-stats" export async function sortLibs(libs: any) { From d48520d75d6521461c114ddb030c86015f79de55 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:50:49 +0200 Subject: [PATCH 07/18] fix sort libraries Signed-off-by: Tuval Simha --- scripts/sort-libraries/sort-libraries.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index af8e81a238..1a58e7601c 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -1,22 +1,21 @@ import { getGemStats } from "./get-gem-stats" import { getGitHubStats } from "./get-github-stats" +import { getHttpScore } from "./get-http-score" import { getNpmStats } from "./get-npm-stats" - export async function sortLibs(libs: any) { { let totalStars = 0 const libsWithScores = await Promise.all( libs.map(async lib => { - console.log(`Fetching stats for ${lib.name}`) - console.log(`Fetching gem for ${lib.gem}`) - console.log(`Fetching github for ${lib.github}`) const [npmStats = {}, gemStars = {}, githubStats = {}] = await Promise.all([ lib.npm && (await getNpmStats(lib.npm)), lib.gem && (await getGemStats(lib.gem)), lib.github && (await getGitHubStats(lib.github)), ]) + const httpScore = await getHttpScore(lib.name) + console.log(`lib.name`, lib.name) const result = { ...lib, ...npmStats, @@ -60,4 +59,4 @@ export async function sortLibs(libs: any) { }) return { sortedLibs, totalStars } } -} +} \ No newline at end of file From e3b30d7d10c20bf29a9519234696b5c57cafc614 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:52:58 +0200 Subject: [PATCH 08/18] fix again Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-github-stats.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index b3313a9ea3..6f676c6452 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -1,5 +1,5 @@ -const numbro = require("numbro") -const timeago = require("timeago.js") +import numbro from "numbro" +import timeago from "timeago.js" export async function getGitHubStats(githubRepo: any) { const [owner, repoName] = githubRepo.split("/") @@ -139,4 +139,4 @@ export async function getGitHubStats(githubRepo: any) { lastRelease, formattedLastRelease: lastRelease && timeago.format(lastRelease), } -} \ No newline at end of file +} From 913fb54030d0cffa28f049772921ed46c41cfc5e Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 12:55:27 +0200 Subject: [PATCH 09/18] fix import Signed-off-by: Tuval Simha --- gatsby-node.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gatsby-node.ts b/gatsby-node.ts index 3dd16efb7a..e9284612e2 100644 --- a/gatsby-node.ts +++ b/gatsby-node.ts @@ -4,7 +4,7 @@ import { promisify } from "util" import { readFile } from "fs/promises" import * as globby from "globby" import * as frontmatterParser from "parser-front-matter" -import { sortLibs } from "./scripts/sort-libraries" +import { sortLibs } from "./scripts/sort-libraries/sort-libraries" const parse$ = promisify(frontmatterParser.parse) From 1a53996c2902398a6e068f2d36499951a95b946e Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Thu, 12 Jan 2023 16:13:34 +0200 Subject: [PATCH 10/18] clean some Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-gem-stats.ts | 12 ++---------- scripts/sort-libraries/get-http-score.ts | 5 +---- scripts/sort-libraries/get-npm-stats.ts | 5 +---- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/scripts/sort-libraries/get-gem-stats.ts b/scripts/sort-libraries/get-gem-stats.ts index d3b5b1a9b2..e54eac16c9 100644 --- a/scripts/sort-libraries/get-gem-stats.ts +++ b/scripts/sort-libraries/get-gem-stats.ts @@ -1,13 +1,5 @@ type GemStatsRespone = { - name?: string - downloads?: number - version?: string - version_created_at?: string - version_downloads?: number - platform?: string - authors?: string - info?: string - licenses?: Array + downloads?: number | undefined } export async function getGemStats( @@ -30,4 +22,4 @@ export async function getGemStats( return { downloads: 0 } } return responseJson -} \ No newline at end of file +} diff --git a/scripts/sort-libraries/get-http-score.ts b/scripts/sort-libraries/get-http-score.ts index 31340a1d92..4178f98810 100644 --- a/scripts/sort-libraries/get-http-score.ts +++ b/scripts/sort-libraries/get-http-score.ts @@ -1,8 +1,5 @@ type HttpScoreRespone = { total?: number - ok?: number - warn?: number - error?: number } export async function getHttpScore( @@ -27,4 +24,4 @@ export async function getHttpScore( return { total: 0 } } return responseJson -} \ No newline at end of file +} diff --git a/scripts/sort-libraries/get-npm-stats.ts b/scripts/sort-libraries/get-npm-stats.ts index 8cd33bd6c4..71f78d4834 100644 --- a/scripts/sort-libraries/get-npm-stats.ts +++ b/scripts/sort-libraries/get-npm-stats.ts @@ -1,8 +1,5 @@ type NpmStatsResponse = { downloads?: number - start?: string - end?: string - package?: string } export async function getNpmStats( @@ -27,4 +24,4 @@ export async function getNpmStats( return { downloads: 0 } } return responseJson -} \ No newline at end of file +} From 29ca27a50e3695a87df721d9b231b7011e126310 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Sun, 15 Jan 2023 15:55:33 +0200 Subject: [PATCH 11/18] some changes Signed-off-by: Tuval Simha --- gatsby-node.ts | 7 +++ scripts/sort-libraries/get-gem-stats.ts | 51 ++++++++++++++++------ scripts/sort-libraries/get-github-stats.ts | 1 + scripts/sort-libraries/get-http-score.ts | 6 --- scripts/sort-libraries/get-npm-stats.ts | 28 ++++++------ scripts/sort-libraries/sort-libraries.ts | 30 +++++++------ 6 files changed, 75 insertions(+), 48 deletions(-) diff --git a/gatsby-node.ts b/gatsby-node.ts index e9284612e2..11561dcf55 100644 --- a/gatsby-node.ts +++ b/gatsby-node.ts @@ -186,6 +186,9 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ Object.keys(libraryCategoryMap).map(async libraryCategoryName => { const libraries = libraryCategoryMap[libraryCategoryName] const { sortedLibs, totalStars } = await sortLibs(libraries) + console.log("sortedLibs from node", sortedLibs) + console.log("totalStars from node", totalStars) + libraryCategoryMap[libraryCategoryName] = sortedLibs languageTotalStars += totalStars || 0 }) @@ -195,6 +198,8 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ totalStars: languageTotalStars, categoryMap: libraryCategoryMap, }) + console.log("languageList from node", languageList) + }), ...Object.keys(codeData.ToolsNew).map(async toolName => { const toolCategoryMap = codeData.ToolsNew[toolName] @@ -212,6 +217,8 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ totalStars: toolTotalStars, categoryMap: toolCategoryMap, }) + console.log("toolList from node", toolList) + }), ]) diff --git a/scripts/sort-libraries/get-gem-stats.ts b/scripts/sort-libraries/get-gem-stats.ts index e54eac16c9..a19dbe047e 100644 --- a/scripts/sort-libraries/get-gem-stats.ts +++ b/scripts/sort-libraries/get-gem-stats.ts @@ -1,25 +1,50 @@ type GemStatsRespone = { - downloads?: number | undefined + name?: string + downloads: number + version?: string + version_created_at?: string + version_downloads?: number + platform?: string + authors?: string + info?: string + licenses?: Array + metadata?: { + homepage_uri?: string + changelog_uri?: string + bug_tracker_uri?: string + source_code_uri?: string + mailing_list_uri?: string + } + yanked?: boolean + sha?: string + gem_uri?: string + homepage_uri?: string + wiki_uri?: string + documentation_uri?: string + mailing_list_uri?: string + source_code_uri?: string + bug_tracker_uri?: string + changelog_uri?: string + funding_uri?: string + dependencies?: { + development?: Array + runtime?: Array + } } -export async function getGemStats( - packageName: string -): Promise { +export async function getGemStats(packageName: string): Promise { + if (!packageName) { + return 0 + } const response = await fetch( `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` ) if (!response.ok) { - console.warn( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return { downloads: 0 } + return 0 } const responseJson: GemStatsRespone = await response.json() if (!responseJson) { - console.warn( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return { downloads: 0 } + return 0 } - return responseJson + return responseJson.downloads } diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index 6f676c6452..39ff30308d 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -85,6 +85,7 @@ export async function getGitHubStats(githubRepo: any) { }, }) const responseJson = await response.json() + if (responseJson && responseJson.errors) { throw JSON.stringify(responseJson.errors) } diff --git a/scripts/sort-libraries/get-http-score.ts b/scripts/sort-libraries/get-http-score.ts index 4178f98810..1cdd2bda8e 100644 --- a/scripts/sort-libraries/get-http-score.ts +++ b/scripts/sort-libraries/get-http-score.ts @@ -11,16 +11,10 @@ export async function getHttpScore( )}/report.json` ) if (!response.ok) { - console.warn( - `getHttpScore: No download count for ${packageName}, so value is 0!` - ) return { total: 0 } } const responseJson: HttpScoreRespone = await response.json() if (!responseJson) { - console.warn( - `getHttpScore: No download count for ${packageName}, so value is 0!` - ) return { total: 0 } } return responseJson diff --git a/scripts/sort-libraries/get-npm-stats.ts b/scripts/sort-libraries/get-npm-stats.ts index 71f78d4834..95165b7ecc 100644 --- a/scripts/sort-libraries/get-npm-stats.ts +++ b/scripts/sort-libraries/get-npm-stats.ts @@ -1,27 +1,25 @@ -type NpmStatsResponse = { - downloads?: number +type NpmStatsType = { + downloads: number + start?: string + end?: string + package?: string } -export async function getNpmStats( - packageName: string -): Promise { +export async function getNpmStats(packageName: string): Promise { + if (!packageName) { + return 0 + } const response = await fetch( `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( packageName )}` ) if (!response.ok) { - console.warn( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return { downloads: 0 } + return 0 } - const responseJson: NpmStatsResponse = await response.json() + const responseJson: NpmStatsType = await response.json() if (!responseJson) { - console.warn( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return { downloads: 0 } + return 0 } - return responseJson + return responseJson.downloads } diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index 1a58e7601c..5683e62e69 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -4,35 +4,35 @@ import { getHttpScore } from "./get-http-score" import { getNpmStats } from "./get-npm-stats" export async function sortLibs(libs: any) { + console.log("libs", libs) + console.log("libs.length", libs.length) { let totalStars = 0 const libsWithScores = await Promise.all( libs.map(async lib => { - const [npmStats = {}, gemStars = {}, githubStats = {}] = - await Promise.all([ - lib.npm && (await getNpmStats(lib.npm)), - lib.gem && (await getGemStats(lib.gem)), - lib.github && (await getGitHubStats(lib.github)), - ]) - const httpScore = await getHttpScore(lib.name) - console.log(`lib.name`, lib.name) + const npmStats = lib.npm && (await getNpmStats(lib.npm)) + const githubStats = lib.github && (await getGitHubStats(lib.github)) + const gemStars = lib.gem && (await getGemStats(lib.gem)) const result = { ...lib, - ...npmStats, - ...gemStars, + npmStats, + gemStars, ...githubStats, } totalStars += result.stars || 0 + console.log("result", result) return result }) ) + console.log("libsWithScores", libsWithScores) + console.log("libsWithScores length", libsWithScores.length) const sortedLibs = libsWithScores.sort((a, b) => { let aScore = 0, bScore = 0 - if ("downloadCount" in a && "downloadCount" in b) { - if (a.downloadCount > b.downloadCount) { + if ("npmStats" in a && "npmStats" in b) { + if (a.npmStats > b.npmStats) { aScore += 40 - } else if (b.downloadCount > a.downloadCount) { + } else if (b.npmStats > a.npmStats) { bScore += 40 } } @@ -57,6 +57,8 @@ export async function sortLibs(libs: any) { } return 0 }) + console.log("sortedLibs", sortedLibs) + return { sortedLibs, totalStars } } -} \ No newline at end of file +} From 59909026fcfe12ba38d5875b9d29f30e000587d6 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Mon, 16 Jan 2023 12:16:42 +0200 Subject: [PATCH 12/18] Fix some and clean Signed-off-by: Tuval Simha --- gatsby-node.ts | 9 +- scripts/sort-libraries/get-gem-stats.ts | 70 ++++--- scripts/sort-libraries/get-github-stats.ts | 4 +- scripts/sort-libraries/get-http-score.ts | 25 ++- scripts/sort-libraries/get-npm-stats.ts | 35 ++-- scripts/sort-libraries/sort-libraries.ts | 120 ++++++----- scripts/sort-libraries/test.ts | 231 +++++++++++++++++++++ 7 files changed, 374 insertions(+), 120 deletions(-) create mode 100644 scripts/sort-libraries/test.ts diff --git a/gatsby-node.ts b/gatsby-node.ts index 11561dcf55..9640c8e7d7 100644 --- a/gatsby-node.ts +++ b/gatsby-node.ts @@ -185,9 +185,12 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ await Promise.all( Object.keys(libraryCategoryMap).map(async libraryCategoryName => { const libraries = libraryCategoryMap[libraryCategoryName] + console.log( + "libraries from node", + libraryCategoryName, + libraries.length + ) const { sortedLibs, totalStars } = await sortLibs(libraries) - console.log("sortedLibs from node", sortedLibs) - console.log("totalStars from node", totalStars) libraryCategoryMap[libraryCategoryName] = sortedLibs languageTotalStars += totalStars || 0 @@ -199,7 +202,6 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ categoryMap: libraryCategoryMap, }) console.log("languageList from node", languageList) - }), ...Object.keys(codeData.ToolsNew).map(async toolName => { const toolCategoryMap = codeData.ToolsNew[toolName] @@ -218,7 +220,6 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ categoryMap: toolCategoryMap, }) console.log("toolList from node", toolList) - }), ]) diff --git a/scripts/sort-libraries/get-gem-stats.ts b/scripts/sort-libraries/get-gem-stats.ts index a19dbe047e..eb52209180 100644 --- a/scripts/sort-libraries/get-gem-stats.ts +++ b/scripts/sort-libraries/get-gem-stats.ts @@ -1,50 +1,52 @@ -type GemStatsRespone = { - name?: string +type GemStatsFetchRespone = { + name: string downloads: number - version?: string - version_created_at?: string - version_downloads?: number - platform?: string - authors?: string - info?: string - licenses?: Array - metadata?: { - homepage_uri?: string - changelog_uri?: string - bug_tracker_uri?: string - source_code_uri?: string - mailing_list_uri?: string + version: string + version_created_at: string + version_downloads: number + platform: string + authors: string + info: string + licenses: Array + metadata: { + homepage_uri: string + changelog_uri: string + bug_tracker_uri: string + source_code_uri: string + mailing_list_uri: string } - yanked?: boolean - sha?: string - gem_uri?: string - homepage_uri?: string - wiki_uri?: string - documentation_uri?: string - mailing_list_uri?: string - source_code_uri?: string - bug_tracker_uri?: string - changelog_uri?: string - funding_uri?: string - dependencies?: { - development?: Array - runtime?: Array + yanked: boolean + sha: string + gem_uri: string + homepage_uri: string + wiki_uri: string + documentation_uri: string + mailing_list_uri: string + source_code_uri: string + bug_tracker_uri: string + changelog_uri: string + funding_uri: string + dependencies: { + development: Array + runtime: Array } } export async function getGemStats(packageName: string): Promise { - if (!packageName) { - return 0 - } const response = await fetch( `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` ) if (!response.ok) { + console.warn(`Get invalid response from GEM for ${packageName}:`, response) return 0 } - const responseJson: GemStatsRespone = await response.json() + const responseJson: GemStatsFetchRespone = await response.json() if (!responseJson) { + console.warn( + `Get invalid response from GEM for ${packageName}:`, + responseJson + ) return 0 } - return responseJson.downloads + return responseJson.downloads ?? 0 } diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index 39ff30308d..6598cd77c2 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -1,5 +1,5 @@ import numbro from "numbro" -import timeago from "timeago.js" +import { format as timeago } from "timeago.js" export async function getGitHubStats(githubRepo: any) { const [owner, repoName] = githubRepo.split("/") @@ -138,6 +138,6 @@ export async function getGitHubStats(githubRepo: any) { formattedStars, license: repo.licenseInfo && repo.licenseInfo.name, lastRelease, - formattedLastRelease: lastRelease && timeago.format(lastRelease), + formattedLastRelease: lastRelease && timeago(lastRelease), } } diff --git a/scripts/sort-libraries/get-http-score.ts b/scripts/sort-libraries/get-http-score.ts index 1cdd2bda8e..9d9f633610 100644 --- a/scripts/sort-libraries/get-http-score.ts +++ b/scripts/sort-libraries/get-http-score.ts @@ -1,21 +1,28 @@ -type HttpScoreRespone = { - total?: number +type HttpScoreFetchResponse = { + total: number + pass: number + errors: number + warnings: number } -export async function getHttpScore( - packageName: string -): Promise { +export async function getHttpScore(packageName: string): Promise { const response = await fetch( `https://raw.githubusercontent.com/graphql/graphql-http/main/implementations/${encodeURIComponent( packageName )}/report.json` ) if (!response.ok) { - return { total: 0 } + console.warn(`Get invalid response from npm for ${packageName}:`, response) + return 0 } - const responseJson: HttpScoreRespone = await response.json() + const responseJson: HttpScoreFetchResponse = await response.json() if (!responseJson) { - return { total: 0 } + console.warn( + `Get invalid response from npm for ${packageName}:`, + responseJson + ) + + return 0 } - return responseJson + return responseJson.total ?? 0 } diff --git a/scripts/sort-libraries/get-npm-stats.ts b/scripts/sort-libraries/get-npm-stats.ts index 95165b7ecc..acd709056c 100644 --- a/scripts/sort-libraries/get-npm-stats.ts +++ b/scripts/sort-libraries/get-npm-stats.ts @@ -1,25 +1,28 @@ -type NpmStatsType = { - downloads: number - start?: string - end?: string - package?: string -} +type NpmStatsFetchResponse = + | { + downloads?: number + start: string + end: string + package: string + } + | { error: string } export async function getNpmStats(packageName: string): Promise { - if (!packageName) { - return 0 - } const response = await fetch( `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( packageName )}` ) - if (!response.ok) { - return 0 - } - const responseJson: NpmStatsType = await response.json() - if (!responseJson) { - return 0 + if (response.ok) { + const responseJson: NpmStatsFetchResponse = await response.json() + if (responseJson && "downloads" in responseJson) { + return responseJson.downloads ?? 0 + } else { + console.warn( + `Get invalid response from npm for ${packageName}:`, + responseJson + ) + } } - return responseJson.downloads + return 0 } diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index 5683e62e69..d7fad9d87d 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -3,62 +3,72 @@ import { getGitHubStats } from "./get-github-stats" import { getHttpScore } from "./get-http-score" import { getNpmStats } from "./get-npm-stats" -export async function sortLibs(libs: any) { - console.log("libs", libs) - console.log("libs.length", libs.length) - { - let totalStars = 0 - const libsWithScores = await Promise.all( - libs.map(async lib => { - const npmStats = lib.npm && (await getNpmStats(lib.npm)) - const githubStats = lib.github && (await getGitHubStats(lib.github)) - const gemStars = lib.gem && (await getGemStats(lib.gem)) - const result = { - ...lib, - npmStats, - gemStars, - ...githubStats, - } - totalStars += result.stars || 0 - console.log("result", result) - return result - }) - ) - console.log("libsWithScores", libsWithScores) - console.log("libsWithScores length", libsWithScores.length) - const sortedLibs = libsWithScores.sort((a, b) => { - let aScore = 0, - bScore = 0 - if ("npmStats" in a && "npmStats" in b) { - if (a.npmStats > b.npmStats) { - aScore += 40 - } else if (b.npmStats > a.npmStats) { - bScore += 40 - } - } - if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { - aScore += 30 - } - if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { - bScore += 30 - } - if ("stars" in a && "stars" in b) { - if (a.stars > b.stars) { - aScore += 40 - } else if (a.stars < b.stars) { - bScore += 40 - } - } - if (bScore > aScore) { - return 1 - } - if (bScore < aScore) { - return -1 +interface Library { + name: string + description: string + howto: string + url: string + github: string + npm: string | undefined + gem: string | undefined + sourcePath: string +} + +export async function sortLibs( + libraries: Library[] +): Promise<{ sortedLibs: Library[]; totalStars: number }> { + console.log("sortLibs length", libraries.length) + + let totalStars = 0 + const libsWithScores = await Promise.all( + libraries.map(async lib => { + const [npmStats, gemStars, githubStats, httpScore] = await Promise.all([ + lib.npm ? getNpmStats(lib.npm) : undefined, + lib.gem ? getGemStats(lib.gem) : undefined, + lib.github ? getGitHubStats(lib.github) : undefined, + lib.name ? getHttpScore(lib.name) : undefined, + ]) + + const result = { + ...lib, + downloadCount: npmStats ?? gemStars ?? 0, + stars: githubStats?.stars ?? 0, + githubStats, } - return 0 + totalStars += result.stars + console.log("result", result) + return result }) - console.log("sortedLibs", sortedLibs) + ) + console.log("libsWithScores", libsWithScores) + const sortedLibs = libsWithScores.sort((a, b) => { + let aScore = 0, + bScore = 0 + if (a.downloadCount > b.downloadCount) { + aScore += 40 + } else if (b.downloadCount > a.downloadCount) { + bScore += 40 + } + + if (a.githubStats?.hasCommitsInLast3Months) { + aScore += 30 + } + if (b.githubStats?.hasCommitsInLast3Months) { + bScore += 30 + } + if (a.stars > b.stars) { + aScore += 40 + } else if (a.stars < b.stars) { + bScore += 40 + } + if (bScore > aScore) { + return 1 + } + if (bScore < aScore) { + return -1 + } + return 0 + }) - return { sortedLibs, totalStars } - } + return { sortedLibs, totalStars } } diff --git a/scripts/sort-libraries/test.ts b/scripts/sort-libraries/test.ts new file mode 100644 index 0000000000..6c09b96a19 --- /dev/null +++ b/scripts/sort-libraries/test.ts @@ -0,0 +1,231 @@ +const numbro = require("numbro") +const timeago = require("timeago.js") + +const getGitHubStats = async githubRepo => { + const [owner, repoName] = githubRepo.split("/") + const accessToken = process.env.GITHUB_ACCESS_TOKEN + if (!accessToken) { + return { + accessToken: false, + } + } + const query = /* GraphQL */ ` + fragment defaultBranchRefFragment on Ref { + target { + ... on Commit { + history(since: $since) { + edges { + node { + author { + name + } + pushedDate + } + } + } + } + } + } + query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { + repositoryOwner(login: $owner) { + repository(name: $repoName) { + defaultBranchRef { + ...defaultBranchRefFragment + } + stargazers { + totalCount + } + updatedAt + forkCount + pullRequests { + totalCount + } + description + licenseInfo { + name + } + releases(first: 1) { + nodes { + publishedAt + } + } + tags: refs( + refPrefix: "refs/tags/" + first: 1 + orderBy: { field: TAG_COMMIT_DATE, direction: DESC } + ) { + nodes { + name + target { + ... on Tag { + target { + ... on Commit { + pushedDate + } + } + } + } + } + } + } + } + } + ` + const lastMonth = new Date() + lastMonth.setMonth(lastMonth.getMonth() - 3) + const response = await fetch("https://api.github.com/graphql", { + method: "POST", + body: JSON.stringify({ + query, + variables: { owner, repoName, since: lastMonth }, + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }) + const responseJson = await response.json() + if (responseJson && responseJson.errors) { + throw JSON.stringify(responseJson.errors) + } + if (!responseJson || !responseJson.data) { + throw `GitHub returned empty response for ${owner}/${repoName}` + } + const { repositoryOwner } = responseJson.data + if (!repositoryOwner) { + throw `No GitHub user found for ${owner}/${repoName}` + } + const { repository: repo } = repositoryOwner + if (!repo) { + throw `No GitHub repo found ${owner}/${repoName}` + } + const stars = repo.stargazers.totalCount + const commitHistory = repo.defaultBranchRef.target.history.edges + + let hasCommitsInLast3Months = false + commitHistory.forEach(commit => { + if (!commit.node.author.name.match(/bot/i)) { + hasCommitsInLast3Months = true + } + }) + const formattedStars = numbro(stars).format({ + average: true, + }) + + const releases: any = [] + if ( + repo.tags && + repo.tags.nodes && + repo.tags.nodes.length && + repo.tags.nodes[0].target.target && + repo.tags.nodes[0].target.target.pushedDate + ) { + releases.push(repo.tags.nodes[0].target.target.pushedDate) + } + if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { + releases.push(repo.releases.nodes[0].publishedAt) + } + if (owner.includes("graphql")) { + console.log({ releases, repoName }) + } + + const lastRelease = releases.filter(Boolean).sort().reverse()[0] + + return { + hasCommitsInLast3Months, + stars, + formattedStars, + license: repo.licenseInfo && repo.licenseInfo.name, + lastRelease, + formattedLastRelease: lastRelease && timeago.format(lastRelease), + } +} + +async function getNpmStats(packageName: string): Promise { + const response = await fetch( + `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( + packageName + )}` + ) + const responseJson = await response.json() + const downloadCount = responseJson.downloads + if (!downloadCount) { + console.debug( + `getNpmStats: No download count for ${packageName}, so value is 0!` + ) + return 0 + } + return downloadCount +} + +async function getGemStats(packageName: string): Promise { + const response = await fetch( + `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` + ) + const responseJson = await response.json() + const downloadCount = responseJson.downloads + console.debug(`getGemStats: ${downloadCount} for ${packageName}`) + if (!downloadCount) { + console.debug( + `getGemStats: No download count for ${packageName}, so value is 0!` + ) + return 0 + } + return downloadCount +} + +export async function sortLibs(libs: any) { + { + let totalStars = 0 + const libsWithScores = await Promise.all( + libs.map(async lib => { + const [npmStats = {}, gemStars = {}, githubStats = {}] = + await Promise.all([ + lib.npm && (await getNpmStats(lib.npm)), + lib.gem && (await getGemStats(lib.gem)), + lib.github && (await getGitHubStats(lib.github)), + ]) + const result = { + ...lib, + ...npmStats, + ...gemStars, + ...githubStats, + } + totalStars += result.stars || 0 + return result + }) + ) + const sortedLibs = libsWithScores.sort((a, b) => { + let aScore = 0, + bScore = 0 + if ("downloadCount" in a && "downloadCount" in b) { + if (a.downloadCount > b.downloadCount) { + aScore += 40 + } else if (b.downloadCount > a.downloadCount) { + bScore += 40 + } + } + if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { + aScore += 30 + } + if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { + bScore += 30 + } + if ("stars" in a && "stars" in b) { + if (a.stars > b.stars) { + aScore += 40 + } else if (a.stars < b.stars) { + bScore += 40 + } + } + if (bScore > aScore) { + return 1 + } + if (bScore < aScore) { + return -1 + } + return 0 + }) + return { sortedLibs, totalStars } + } +} From cebda5f4d47ffc36051f0d13d6c96fabe8ed7d15 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Mon, 16 Jan 2023 12:17:31 +0200 Subject: [PATCH 13/18] delete test ts Signed-off-by: Tuval Simha --- scripts/sort-libraries/test.ts | 231 --------------------------------- 1 file changed, 231 deletions(-) delete mode 100644 scripts/sort-libraries/test.ts diff --git a/scripts/sort-libraries/test.ts b/scripts/sort-libraries/test.ts deleted file mode 100644 index 6c09b96a19..0000000000 --- a/scripts/sort-libraries/test.ts +++ /dev/null @@ -1,231 +0,0 @@ -const numbro = require("numbro") -const timeago = require("timeago.js") - -const getGitHubStats = async githubRepo => { - const [owner, repoName] = githubRepo.split("/") - const accessToken = process.env.GITHUB_ACCESS_TOKEN - if (!accessToken) { - return { - accessToken: false, - } - } - const query = /* GraphQL */ ` - fragment defaultBranchRefFragment on Ref { - target { - ... on Commit { - history(since: $since) { - edges { - node { - author { - name - } - pushedDate - } - } - } - } - } - } - query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { - repositoryOwner(login: $owner) { - repository(name: $repoName) { - defaultBranchRef { - ...defaultBranchRefFragment - } - stargazers { - totalCount - } - updatedAt - forkCount - pullRequests { - totalCount - } - description - licenseInfo { - name - } - releases(first: 1) { - nodes { - publishedAt - } - } - tags: refs( - refPrefix: "refs/tags/" - first: 1 - orderBy: { field: TAG_COMMIT_DATE, direction: DESC } - ) { - nodes { - name - target { - ... on Tag { - target { - ... on Commit { - pushedDate - } - } - } - } - } - } - } - } - } - ` - const lastMonth = new Date() - lastMonth.setMonth(lastMonth.getMonth() - 3) - const response = await fetch("https://api.github.com/graphql", { - method: "POST", - body: JSON.stringify({ - query, - variables: { owner, repoName, since: lastMonth }, - }), - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }) - const responseJson = await response.json() - if (responseJson && responseJson.errors) { - throw JSON.stringify(responseJson.errors) - } - if (!responseJson || !responseJson.data) { - throw `GitHub returned empty response for ${owner}/${repoName}` - } - const { repositoryOwner } = responseJson.data - if (!repositoryOwner) { - throw `No GitHub user found for ${owner}/${repoName}` - } - const { repository: repo } = repositoryOwner - if (!repo) { - throw `No GitHub repo found ${owner}/${repoName}` - } - const stars = repo.stargazers.totalCount - const commitHistory = repo.defaultBranchRef.target.history.edges - - let hasCommitsInLast3Months = false - commitHistory.forEach(commit => { - if (!commit.node.author.name.match(/bot/i)) { - hasCommitsInLast3Months = true - } - }) - const formattedStars = numbro(stars).format({ - average: true, - }) - - const releases: any = [] - if ( - repo.tags && - repo.tags.nodes && - repo.tags.nodes.length && - repo.tags.nodes[0].target.target && - repo.tags.nodes[0].target.target.pushedDate - ) { - releases.push(repo.tags.nodes[0].target.target.pushedDate) - } - if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { - releases.push(repo.releases.nodes[0].publishedAt) - } - if (owner.includes("graphql")) { - console.log({ releases, repoName }) - } - - const lastRelease = releases.filter(Boolean).sort().reverse()[0] - - return { - hasCommitsInLast3Months, - stars, - formattedStars, - license: repo.licenseInfo && repo.licenseInfo.name, - lastRelease, - formattedLastRelease: lastRelease && timeago.format(lastRelease), - } -} - -async function getNpmStats(packageName: string): Promise { - const response = await fetch( - `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( - packageName - )}` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - if (!downloadCount) { - console.debug( - `getNpmStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} - -async function getGemStats(packageName: string): Promise { - const response = await fetch( - `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - console.debug(`getGemStats: ${downloadCount} for ${packageName}`) - if (!downloadCount) { - console.debug( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} - -export async function sortLibs(libs: any) { - { - let totalStars = 0 - const libsWithScores = await Promise.all( - libs.map(async lib => { - const [npmStats = {}, gemStars = {}, githubStats = {}] = - await Promise.all([ - lib.npm && (await getNpmStats(lib.npm)), - lib.gem && (await getGemStats(lib.gem)), - lib.github && (await getGitHubStats(lib.github)), - ]) - const result = { - ...lib, - ...npmStats, - ...gemStars, - ...githubStats, - } - totalStars += result.stars || 0 - return result - }) - ) - const sortedLibs = libsWithScores.sort((a, b) => { - let aScore = 0, - bScore = 0 - if ("downloadCount" in a && "downloadCount" in b) { - if (a.downloadCount > b.downloadCount) { - aScore += 40 - } else if (b.downloadCount > a.downloadCount) { - bScore += 40 - } - } - if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { - aScore += 30 - } - if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { - bScore += 30 - } - if ("stars" in a && "stars" in b) { - if (a.stars > b.stars) { - aScore += 40 - } else if (a.stars < b.stars) { - bScore += 40 - } - } - if (bScore > aScore) { - return 1 - } - if (bScore < aScore) { - return -1 - } - return 0 - }) - return { sortedLibs, totalStars } - } -} From 72e79cc2ede0a4f701ddaf4d70242a76ab87a533 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Mon, 16 Jan 2023 12:44:30 +0200 Subject: [PATCH 14/18] clean some Signed-off-by: Tuval Simha --- gatsby-node.ts | 7 ------- scripts/sort-libraries/get-github-stats.ts | 16 ++++++++++----- scripts/sort-libraries/get-http-score.ts | 10 ++++------ scripts/sort-libraries/get-npm-stats.ts | 5 +---- scripts/sort-libraries/sort-libraries.ts | 23 ++++++++++++---------- 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/gatsby-node.ts b/gatsby-node.ts index 9640c8e7d7..dcf634af31 100644 --- a/gatsby-node.ts +++ b/gatsby-node.ts @@ -185,11 +185,6 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ await Promise.all( Object.keys(libraryCategoryMap).map(async libraryCategoryName => { const libraries = libraryCategoryMap[libraryCategoryName] - console.log( - "libraries from node", - libraryCategoryName, - libraries.length - ) const { sortedLibs, totalStars } = await sortLibs(libraries) libraryCategoryMap[libraryCategoryName] = sortedLibs @@ -201,7 +196,6 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ totalStars: languageTotalStars, categoryMap: libraryCategoryMap, }) - console.log("languageList from node", languageList) }), ...Object.keys(codeData.ToolsNew).map(async toolName => { const toolCategoryMap = codeData.ToolsNew[toolName] @@ -219,7 +213,6 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ totalStars: toolTotalStars, categoryMap: toolCategoryMap, }) - console.log("toolList from node", toolList) }), ]) diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index 6598cd77c2..95f07dbbd4 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -1,13 +1,13 @@ import numbro from "numbro" import { format as timeago } from "timeago.js" -export async function getGitHubStats(githubRepo: any) { +export async function getGitHubStats(githubRepo: string) { const [owner, repoName] = githubRepo.split("/") const accessToken = process.env.GITHUB_ACCESS_TOKEN if (!accessToken) { - return { - accessToken: false, - } + return console.warn( + `No GITHUB_ACCESS_TOKEN environment variable found. Skipping GitHub stats for ${githubRepo}` + ) } const query = /* GraphQL */ ` fragment defaultBranchRefFragment on Ref { @@ -26,7 +26,11 @@ export async function getGitHubStats(githubRepo: any) { } } } - query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { + query GitHubInfo( + $owner: String! + $repoName: String! + $since: GitTimestamp! + ) { repositoryOwner(login: $owner) { repository(name: $repoName) { defaultBranchRef { @@ -84,7 +88,9 @@ export async function getGitHubStats(githubRepo: any) { "Content-Type": "application/json", }, }) + console.log("response:", response) const responseJson = await response.json() + console.log("responseJson:", responseJson) if (responseJson && responseJson.errors) { throw JSON.stringify(responseJson.errors) diff --git a/scripts/sort-libraries/get-http-score.ts b/scripts/sort-libraries/get-http-score.ts index 9d9f633610..375c142834 100644 --- a/scripts/sort-libraries/get-http-score.ts +++ b/scripts/sort-libraries/get-http-score.ts @@ -12,16 +12,14 @@ export async function getHttpScore(packageName: string): Promise { )}/report.json` ) if (!response.ok) { - console.warn(`Get invalid response from npm for ${packageName}:`, response) + console.warn( + `Get invalid response from HTTP score for ${packageName}. Status: ${response.status}` + ) return 0 } const responseJson: HttpScoreFetchResponse = await response.json() if (!responseJson) { - console.warn( - `Get invalid response from npm for ${packageName}:`, - responseJson - ) - + console.warn(`Get invalid response from HTTP score for ${packageName}`) return 0 } return responseJson.total ?? 0 diff --git a/scripts/sort-libraries/get-npm-stats.ts b/scripts/sort-libraries/get-npm-stats.ts index acd709056c..93b5529d55 100644 --- a/scripts/sort-libraries/get-npm-stats.ts +++ b/scripts/sort-libraries/get-npm-stats.ts @@ -18,10 +18,7 @@ export async function getNpmStats(packageName: string): Promise { if (responseJson && "downloads" in responseJson) { return responseJson.downloads ?? 0 } else { - console.warn( - `Get invalid response from npm for ${packageName}:`, - responseJson - ) + console.warn(`Get invalid response from npm for ${packageName}`) } } return 0 diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index d7fad9d87d..5c7d82ba67 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -17,8 +17,6 @@ interface Library { export async function sortLibs( libraries: Library[] ): Promise<{ sortedLibs: Library[]; totalStars: number }> { - console.log("sortLibs length", libraries.length) - let totalStars = 0 const libsWithScores = await Promise.all( libraries.map(async lib => { @@ -33,33 +31,38 @@ export async function sortLibs( ...lib, downloadCount: npmStats ?? gemStars ?? 0, stars: githubStats?.stars ?? 0, + httpScore: httpScore ?? 0, githubStats, } totalStars += result.stars - console.log("result", result) return result }) ) - console.log("libsWithScores", libsWithScores) const sortedLibs = libsWithScores.sort((a, b) => { let aScore = 0, bScore = 0 if (a.downloadCount > b.downloadCount) { - aScore += 40 + aScore += 36 } else if (b.downloadCount > a.downloadCount) { - bScore += 40 + bScore += 36 + } + + if (a.httpScore > b.httpScore) { + aScore += 10 + } else if (b.httpScore > a.httpScore) { + bScore += 10 } if (a.githubStats?.hasCommitsInLast3Months) { - aScore += 30 + aScore += 28 } if (b.githubStats?.hasCommitsInLast3Months) { - bScore += 30 + bScore += 28 } if (a.stars > b.stars) { - aScore += 40 + aScore += 36 } else if (a.stars < b.stars) { - bScore += 40 + bScore += 36 } if (bScore > aScore) { return 1 From 8b5d173a92cbcee6f7fe1267a6e874dc781f38f8 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Mon, 16 Jan 2023 16:14:58 +0200 Subject: [PATCH 15/18] clean Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-github-stats.ts | 187 +++++++++++++++------ scripts/sort-libraries/sort-libraries.ts | 6 +- 2 files changed, 138 insertions(+), 55 deletions(-) diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index 95f07dbbd4..2fc201b407 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -1,6 +1,85 @@ import numbro from "numbro" import { format as timeago } from "timeago.js" +type GitHubStatsFetchResponse = + | { + errors: [ + { + extensions: { + value: string + problems: [ + { + path: string + explanation: string + } + ] + } + locations: [ + { + line: number + column: number + } + ] + message: string + } + ] + } + | { + data: { + repositoryOwner: { + repository: { + defaultBranchRef: { + target: { + history: { + edges: [ + { + node: { + author: { + name: string + } + pushedDate: string + } + } + ] + } + } + } + stargazers: { + totalCount: number + } + updatedAt: string + forkCount: number + pullRequests: { + totalCount: number + } + description: string + licenseInfo: { + name: string + } + releases: { + nodes: [ + { + publishedAt: string + } + ] + } + tags: { + nodes: [ + { + name: string + target: { + target: { + pushedDate: string + } + } + } + ] + } + } + } + } + } + export async function getGitHubStats(githubRepo: string) { const [owner, repoName] = githubRepo.split("/") const accessToken = process.env.GITHUB_ACCESS_TOKEN @@ -88,62 +167,66 @@ export async function getGitHubStats(githubRepo: string) { "Content-Type": "application/json", }, }) - console.log("response:", response) - const responseJson = await response.json() - console.log("responseJson:", responseJson) - - if (responseJson && responseJson.errors) { - throw JSON.stringify(responseJson.errors) - } - if (!responseJson || !responseJson.data) { - throw `GitHub returned empty response for ${owner}/${repoName}` - } - const { repositoryOwner } = responseJson.data - if (!repositoryOwner) { - throw `No GitHub user found for ${owner}/${repoName}` - } - const { repository: repo } = repositoryOwner - if (!repo) { - throw `No GitHub repo found ${owner}/${repoName}` + if (!response.ok) { + console.warn( + `Get invalid response from GitHub for ${owner}/${repoName}. Status: ${response.status}` + ) } - const stars = repo.stargazers.totalCount - const commitHistory = repo.defaultBranchRef.target.history.edges + const responseJson: GitHubStatsFetchResponse = await response.json() - let hasCommitsInLast3Months = false - commitHistory.forEach(commit => { - if (!commit.node.author.name.match(/bot/i)) { - hasCommitsInLast3Months = true + if (responseJson && "data" in responseJson) { + const repositoryOwner = responseJson.data.repositoryOwner + if (!repositoryOwner) { + throw `No GitHub user found for ${owner}/${repoName}` } - }) - const formattedStars = numbro(stars).format({ - average: true, - }) - - const releases: any = [] - if ( - repo.tags && - repo.tags.nodes && - repo.tags.nodes.length && - repo.tags.nodes[0].target.target && - repo.tags.nodes[0].target.target.pushedDate - ) { - releases.push(repo.tags.nodes[0].target.target.pushedDate) - } - if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { - releases.push(repo.releases.nodes[0].publishedAt) - } - if (owner.includes("graphql")) { - console.log({ releases, repoName }) - } + const { repository: repo } = repositoryOwner + console.log("repo:", repo.tags) + if (!repo) { + throw `No GitHub repo found ${owner}/${repoName}` + } + const stars = repo.stargazers.totalCount + const commitHistory = repo.defaultBranchRef.target.history.edges - const lastRelease = releases.filter(Boolean).sort().reverse()[0] + let hasCommitsInLast3Months = false + commitHistory.forEach(commit => { + if (!commit.node.author.name.match(/bot/i)) { + hasCommitsInLast3Months = true + } + }) + const formattedStars = numbro(stars).format({ + average: true, + }) + const releases: any = [] + if ( + repo.tags && + repo.tags.nodes && + repo.tags.nodes.length && + repo.tags.nodes[0].target.target && + repo.tags.nodes[0].target.target.pushedDate + ) { + releases.push(repo.tags.nodes[0].target.target.pushedDate) + } + if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { + releases.push(repo.releases.nodes[0].publishedAt) + } + if (owner.includes("graphql")) { + console.log({ releases, repoName }) + } - return { - hasCommitsInLast3Months, - stars, - formattedStars, - license: repo.licenseInfo && repo.licenseInfo.name, - lastRelease, - formattedLastRelease: lastRelease && timeago(lastRelease), + const lastRelease = releases.filter(Boolean).sort().reverse()[0] + return { + hasCommitsInLast3Months, + stars, + formattedStars, + license: repo.licenseInfo && repo.licenseInfo.name, + lastRelease, + formattedLastRelease: lastRelease && timeago(lastRelease), + } + } else { + console.warn( + `Get invalid response from GitHub for ${owner}/${repoName}. Response: ${JSON.stringify( + responseJson + )}` + ) } } diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts index 5c7d82ba67..4c297856ca 100644 --- a/scripts/sort-libraries/sort-libraries.ts +++ b/scripts/sort-libraries/sort-libraries.ts @@ -32,7 +32,7 @@ export async function sortLibs( downloadCount: npmStats ?? gemStars ?? 0, stars: githubStats?.stars ?? 0, httpScore: httpScore ?? 0, - githubStats, + ...githubStats, } totalStars += result.stars return result @@ -53,10 +53,10 @@ export async function sortLibs( bScore += 10 } - if (a.githubStats?.hasCommitsInLast3Months) { + if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { aScore += 28 } - if (b.githubStats?.hasCommitsInLast3Months) { + if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { bScore += 28 } if (a.stars > b.stars) { From 4441444067d5204337eb5d41e880b6666a18bd4a Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Mon, 16 Jan 2023 16:19:56 +0200 Subject: [PATCH 16/18] type promise Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-github-stats.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index 2fc201b407..af193761f2 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -80,13 +80,25 @@ type GitHubStatsFetchResponse = } } -export async function getGitHubStats(githubRepo: string) { +type GitHubInfo = { + hasCommitsInLast3Months: boolean + stars: number + formattedStars: string + license: string + lastRelease: string + formattedLastRelease: string +} + +export async function getGitHubStats( + githubRepo: string +): Promise { const [owner, repoName] = githubRepo.split("/") const accessToken = process.env.GITHUB_ACCESS_TOKEN if (!accessToken) { - return console.warn( + console.warn( `No GITHUB_ACCESS_TOKEN environment variable found. Skipping GitHub stats for ${githubRepo}` ) + return } const query = /* GraphQL */ ` fragment defaultBranchRefFragment on Ref { From d3a1417163ce2424491517eb960b61c95c496ab1 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Tue, 17 Jan 2023 13:15:45 +0200 Subject: [PATCH 17/18] add Release type Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-github-stats.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index af193761f2..27e11986ab 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -89,6 +89,8 @@ type GitHubInfo = { formattedLastRelease: string } +type Release = { date: string; formattedDate: string } + export async function getGitHubStats( githubRepo: string ): Promise { @@ -208,7 +210,7 @@ export async function getGitHubStats( const formattedStars = numbro(stars).format({ average: true, }) - const releases: any = [] + const releases: Release[] = [] if ( repo.tags && repo.tags.nodes && @@ -216,23 +218,31 @@ export async function getGitHubStats( repo.tags.nodes[0].target.target && repo.tags.nodes[0].target.target.pushedDate ) { - releases.push(repo.tags.nodes[0].target.target.pushedDate) + releases.push({ + date: repo.tags.nodes[0].target.target.pushedDate, + formattedDate: timeago(repo.tags.nodes[0].target.target.pushedDate), + }) } if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { - releases.push(repo.releases.nodes[0].publishedAt) + releases.push({ + date: repo.releases.nodes[0].publishedAt, + formattedDate: timeago(repo.releases.nodes[0].publishedAt), + }) } if (owner.includes("graphql")) { console.log({ releases, repoName }) } + console.log("releases", releases) const lastRelease = releases.filter(Boolean).sort().reverse()[0] + console.log("lastRelease", lastRelease) return { hasCommitsInLast3Months, stars, formattedStars, license: repo.licenseInfo && repo.licenseInfo.name, - lastRelease, - formattedLastRelease: lastRelease && timeago(lastRelease), + lastRelease: lastRelease ? lastRelease.date : "", + formattedLastRelease: lastRelease ? lastRelease.formattedDate : "", } } else { console.warn( From b4a75fed7389859d68a370fe9e617cc5fcd2f3c6 Mon Sep 17 00:00:00 2001 From: Tuval Simha Date: Mon, 23 Jan 2023 10:44:46 +0200 Subject: [PATCH 18/18] clean? Signed-off-by: Tuval Simha --- scripts/sort-libraries/get-github-stats.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts index 27e11986ab..88fc1eef0d 100644 --- a/scripts/sort-libraries/get-github-stats.ts +++ b/scripts/sort-libraries/get-github-stats.ts @@ -100,7 +100,7 @@ export async function getGitHubStats( console.warn( `No GITHUB_ACCESS_TOKEN environment variable found. Skipping GitHub stats for ${githubRepo}` ) - return + return undefined } const query = /* GraphQL */ ` fragment defaultBranchRefFragment on Ref { @@ -235,7 +235,6 @@ export async function getGitHubStats( console.log("releases", releases) const lastRelease = releases.filter(Boolean).sort().reverse()[0] - console.log("lastRelease", lastRelease) return { hasCommitsInLast3Months, stars,