diff --git a/gatsby-node.ts b/gatsby-node.ts index 3dd16efb7a..dcf634af31 100644 --- a/gatsby-node.ts +++ b/gatsby-node.ts @@ -4,7 +4,7 @@ import { promisify } from "util" import { readFile } from "fs/promises" import * as globby from "globby" import * as frontmatterParser from "parser-front-matter" -import { sortLibs } from "./scripts/sort-libraries" +import { sortLibs } from "./scripts/sort-libraries/sort-libraries" const parse$ = promisify(frontmatterParser.parse) @@ -186,6 +186,7 @@ export const onCreatePage: GatsbyNode["onCreatePage"] = async ({ Object.keys(libraryCategoryMap).map(async libraryCategoryName => { const libraries = libraryCategoryMap[libraryCategoryName] const { sortedLibs, totalStars } = await sortLibs(libraries) + libraryCategoryMap[libraryCategoryName] = sortedLibs languageTotalStars += totalStars || 0 }) diff --git a/scripts/sort-libraries.ts b/scripts/sort-libraries.ts deleted file mode 100644 index bf053346c1..0000000000 --- a/scripts/sort-libraries.ts +++ /dev/null @@ -1,234 +0,0 @@ -const numbro = require("numbro") -const timeago = require("timeago.js") - -const getGitHubStats = async githubRepo => { - const [owner, repoName] = githubRepo.split("/") - const accessToken = process.env.GITHUB_ACCESS_TOKEN - if (!accessToken) { - return { - accessToken: false, - } - } - const query = /* GraphQL */ ` - fragment defaultBranchRefFragment on Ref { - target { - ... on Commit { - history(since: $since) { - edges { - node { - author { - name - } - pushedDate - } - } - } - } - } - } - query ($owner: String!, $repoName: String!, $since: GitTimestamp!) { - repositoryOwner(login: $owner) { - repository(name: $repoName) { - defaultBranchRef { - ...defaultBranchRefFragment - } - stargazers { - totalCount - } - updatedAt - forkCount - pullRequests { - totalCount - } - description - licenseInfo { - name - } - releases(first: 1) { - nodes { - publishedAt - } - } - tags: refs( - refPrefix: "refs/tags/" - first: 1 - orderBy: { field: TAG_COMMIT_DATE, direction: DESC } - ) { - nodes { - name - target { - ... on Tag { - target { - ... on Commit { - pushedDate - } - } - } - } - } - } - } - } - } - ` - const lastMonth = new Date() - lastMonth.setMonth(lastMonth.getMonth() - 3) - const response = await fetch("https://api.github.com/graphql", { - method: "POST", - body: JSON.stringify({ - query, - variables: { owner, repoName, since: lastMonth }, - }), - headers: { - Authorization: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }) - const responseJson = await response.json() - if (responseJson && responseJson.errors) { - throw JSON.stringify(responseJson.errors) - } - if (!responseJson || !responseJson.data) { - throw `GitHub returned empty response for ${owner}/${repoName}` - } - const { repositoryOwner } = responseJson.data - if (!repositoryOwner) { - throw `No GitHub user found for ${owner}/${repoName}` - } - const { repository: repo } = repositoryOwner - if (!repo) { - throw `No GitHub repo found ${owner}/${repoName}` - } - const stars = repo.stargazers.totalCount - const commitHistory = repo.defaultBranchRef.target.history.edges - - let hasCommitsInLast3Months = false - commitHistory.forEach(commit => { - if (!commit.node.author.name.match(/bot/i)) { - hasCommitsInLast3Months = true - } - }) - const formattedStars = numbro(stars).format({ - average: true, - }) - - const releases: any = [] - if ( - repo.tags && - repo.tags.nodes && - repo.tags.nodes.length && - repo.tags.nodes[0].target.target && - repo.tags.nodes[0].target.target.pushedDate - ) { - releases.push(repo.tags.nodes[0].target.target.pushedDate) - } - if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { - releases.push(repo.releases.nodes[0].publishedAt) - } - if (owner.includes("graphql")) { - console.log({ releases, repoName }) - } - - const lastRelease = releases.filter(Boolean).sort().reverse()[0] - - return { - hasCommitsInLast3Months, - stars, - formattedStars, - license: repo.licenseInfo && repo.licenseInfo.name, - lastRelease, - formattedLastRelease: lastRelease && timeago.format(lastRelease), - } -} - -async function getNpmStats(packageName: string): Promise { - const response = await fetch( - `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( - packageName - )}` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - if (!downloadCount) { - console.debug( - `getNpmStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} - -async function getGemStats(packageName: string): Promise { - const response = await fetch( - `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` - ) - const responseJson = await response.json() - const downloadCount = responseJson.downloads - console.debug(`getGemStats: ${downloadCount} for ${packageName}`) - if (!downloadCount) { - console.debug( - `getGemStats: No download count for ${packageName}, so value is 0!` - ) - return 0 - } - return downloadCount -} - -export async function sortLibs(libs: any) { - { - let totalStars = 0 - const libsWithScores = await Promise.all( - libs.map(async lib => { - console.log(`Fetching stats for ${lib.name}`) - console.log(`Fetching gem for ${lib.gem}`) - console.log(`Fetching github for ${lib.github}`) - const [npmStats = {}, gemStars = {}, githubStats = {}] = - await Promise.all([ - lib.npm && (await getNpmStats(lib.npm)), - lib.gem && (await getGemStats(lib.gem)), - lib.github && (await getGitHubStats(lib.github)), - ]) - const result = { - ...lib, - ...npmStats, - ...gemStars, - ...githubStats, - } - totalStars += result.stars || 0 - return result - }) - ) - const sortedLibs = libsWithScores.sort((a, b) => { - let aScore = 0, - bScore = 0 - if ("downloadCount" in a && "downloadCount" in b) { - if (a.downloadCount > b.downloadCount) { - aScore += 40 - } else if (b.downloadCount > a.downloadCount) { - bScore += 40 - } - } - if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { - aScore += 30 - } - if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { - bScore += 30 - } - if ("stars" in a && "stars" in b) { - if (a.stars > b.stars) { - aScore += 40 - } else if (a.stars < b.stars) { - bScore += 40 - } - } - if (bScore > aScore) { - return 1 - } - if (bScore < aScore) { - return -1 - } - return 0 - }) - return { sortedLibs, totalStars } - } -} diff --git a/scripts/sort-libraries/get-gem-stats.ts b/scripts/sort-libraries/get-gem-stats.ts new file mode 100644 index 0000000000..eb52209180 --- /dev/null +++ b/scripts/sort-libraries/get-gem-stats.ts @@ -0,0 +1,52 @@ +type GemStatsFetchRespone = { + name: string + downloads: number + version: string + version_created_at: string + version_downloads: number + platform: string + authors: string + info: string + licenses: Array + metadata: { + homepage_uri: string + changelog_uri: string + bug_tracker_uri: string + source_code_uri: string + mailing_list_uri: string + } + yanked: boolean + sha: string + gem_uri: string + homepage_uri: string + wiki_uri: string + documentation_uri: string + mailing_list_uri: string + source_code_uri: string + bug_tracker_uri: string + changelog_uri: string + funding_uri: string + dependencies: { + development: Array + runtime: Array + } +} + +export async function getGemStats(packageName: string): Promise { + const response = await fetch( + `https://rubygems.org/api/v1/gems/${encodeURIComponent(packageName)}.json` + ) + if (!response.ok) { + console.warn(`Get invalid response from GEM for ${packageName}:`, response) + return 0 + } + const responseJson: GemStatsFetchRespone = await response.json() + if (!responseJson) { + console.warn( + `Get invalid response from GEM for ${packageName}:`, + responseJson + ) + return 0 + } + return responseJson.downloads ?? 0 +} diff --git a/scripts/sort-libraries/get-github-stats.ts b/scripts/sort-libraries/get-github-stats.ts new file mode 100644 index 0000000000..88fc1eef0d --- /dev/null +++ b/scripts/sort-libraries/get-github-stats.ts @@ -0,0 +1,253 @@ +import numbro from "numbro" +import { format as timeago } from "timeago.js" + +type GitHubStatsFetchResponse = + | { + errors: [ + { + extensions: { + value: string + problems: [ + { + path: string + explanation: string + } + ] + } + locations: [ + { + line: number + column: number + } + ] + message: string + } + ] + } + | { + data: { + repositoryOwner: { + repository: { + defaultBranchRef: { + target: { + history: { + edges: [ + { + node: { + author: { + name: string + } + pushedDate: string + } + } + ] + } + } + } + stargazers: { + totalCount: number + } + updatedAt: string + forkCount: number + pullRequests: { + totalCount: number + } + description: string + licenseInfo: { + name: string + } + releases: { + nodes: [ + { + publishedAt: string + } + ] + } + tags: { + nodes: [ + { + name: string + target: { + target: { + pushedDate: string + } + } + } + ] + } + } + } + } + } + +type GitHubInfo = { + hasCommitsInLast3Months: boolean + stars: number + formattedStars: string + license: string + lastRelease: string + formattedLastRelease: string +} + +type Release = { date: string; formattedDate: string } + +export async function getGitHubStats( + githubRepo: string +): Promise { + const [owner, repoName] = githubRepo.split("/") + const accessToken = process.env.GITHUB_ACCESS_TOKEN + if (!accessToken) { + console.warn( + `No GITHUB_ACCESS_TOKEN environment variable found. Skipping GitHub stats for ${githubRepo}` + ) + return undefined + } + const query = /* GraphQL */ ` + fragment defaultBranchRefFragment on Ref { + target { + ... on Commit { + history(since: $since) { + edges { + node { + author { + name + } + pushedDate + } + } + } + } + } + } + query GitHubInfo( + $owner: String! + $repoName: String! + $since: GitTimestamp! + ) { + repositoryOwner(login: $owner) { + repository(name: $repoName) { + defaultBranchRef { + ...defaultBranchRefFragment + } + stargazers { + totalCount + } + updatedAt + forkCount + pullRequests { + totalCount + } + description + licenseInfo { + name + } + releases(first: 1) { + nodes { + publishedAt + } + } + tags: refs( + refPrefix: "refs/tags/" + first: 1 + orderBy: { field: TAG_COMMIT_DATE, direction: DESC } + ) { + nodes { + name + target { + ... on Tag { + target { + ... on Commit { + pushedDate + } + } + } + } + } + } + } + } + } + ` + const lastMonth = new Date() + lastMonth.setMonth(lastMonth.getMonth() - 3) + const response = await fetch("https://api.github.com/graphql", { + method: "POST", + body: JSON.stringify({ + query, + variables: { owner, repoName, since: lastMonth }, + }), + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }) + if (!response.ok) { + console.warn( + `Get invalid response from GitHub for ${owner}/${repoName}. Status: ${response.status}` + ) + } + const responseJson: GitHubStatsFetchResponse = await response.json() + + if (responseJson && "data" in responseJson) { + const repositoryOwner = responseJson.data.repositoryOwner + if (!repositoryOwner) { + throw `No GitHub user found for ${owner}/${repoName}` + } + const { repository: repo } = repositoryOwner + console.log("repo:", repo.tags) + if (!repo) { + throw `No GitHub repo found ${owner}/${repoName}` + } + const stars = repo.stargazers.totalCount + const commitHistory = repo.defaultBranchRef.target.history.edges + + let hasCommitsInLast3Months = false + commitHistory.forEach(commit => { + if (!commit.node.author.name.match(/bot/i)) { + hasCommitsInLast3Months = true + } + }) + const formattedStars = numbro(stars).format({ + average: true, + }) + const releases: Release[] = [] + if ( + repo.tags && + repo.tags.nodes && + repo.tags.nodes.length && + repo.tags.nodes[0].target.target && + repo.tags.nodes[0].target.target.pushedDate + ) { + releases.push({ + date: repo.tags.nodes[0].target.target.pushedDate, + formattedDate: timeago(repo.tags.nodes[0].target.target.pushedDate), + }) + } + if (repo.releases && repo.releases.nodes && repo.releases.nodes.length) { + releases.push({ + date: repo.releases.nodes[0].publishedAt, + formattedDate: timeago(repo.releases.nodes[0].publishedAt), + }) + } + if (owner.includes("graphql")) { + console.log({ releases, repoName }) + } + console.log("releases", releases) + + const lastRelease = releases.filter(Boolean).sort().reverse()[0] + return { + hasCommitsInLast3Months, + stars, + formattedStars, + license: repo.licenseInfo && repo.licenseInfo.name, + lastRelease: lastRelease ? lastRelease.date : "", + formattedLastRelease: lastRelease ? lastRelease.formattedDate : "", + } + } else { + console.warn( + `Get invalid response from GitHub for ${owner}/${repoName}. Response: ${JSON.stringify( + responseJson + )}` + ) + } +} diff --git a/scripts/sort-libraries/get-http-score.ts b/scripts/sort-libraries/get-http-score.ts new file mode 100644 index 0000000000..375c142834 --- /dev/null +++ b/scripts/sort-libraries/get-http-score.ts @@ -0,0 +1,26 @@ +type HttpScoreFetchResponse = { + total: number + pass: number + errors: number + warnings: number +} + +export async function getHttpScore(packageName: string): Promise { + const response = await fetch( + `https://raw.githubusercontent.com/graphql/graphql-http/main/implementations/${encodeURIComponent( + packageName + )}/report.json` + ) + if (!response.ok) { + console.warn( + `Get invalid response from HTTP score for ${packageName}. Status: ${response.status}` + ) + return 0 + } + const responseJson: HttpScoreFetchResponse = await response.json() + if (!responseJson) { + console.warn(`Get invalid response from HTTP score for ${packageName}`) + return 0 + } + return responseJson.total ?? 0 +} diff --git a/scripts/sort-libraries/get-npm-stats.ts b/scripts/sort-libraries/get-npm-stats.ts new file mode 100644 index 0000000000..93b5529d55 --- /dev/null +++ b/scripts/sort-libraries/get-npm-stats.ts @@ -0,0 +1,25 @@ +type NpmStatsFetchResponse = + | { + downloads?: number + start: string + end: string + package: string + } + | { error: string } + +export async function getNpmStats(packageName: string): Promise { + const response = await fetch( + `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent( + packageName + )}` + ) + if (response.ok) { + const responseJson: NpmStatsFetchResponse = await response.json() + if (responseJson && "downloads" in responseJson) { + return responseJson.downloads ?? 0 + } else { + console.warn(`Get invalid response from npm for ${packageName}`) + } + } + return 0 +} diff --git a/scripts/sort-libraries/sort-libraries.ts b/scripts/sort-libraries/sort-libraries.ts new file mode 100644 index 0000000000..4c297856ca --- /dev/null +++ b/scripts/sort-libraries/sort-libraries.ts @@ -0,0 +1,77 @@ +import { getGemStats } from "./get-gem-stats" +import { getGitHubStats } from "./get-github-stats" +import { getHttpScore } from "./get-http-score" +import { getNpmStats } from "./get-npm-stats" + +interface Library { + name: string + description: string + howto: string + url: string + github: string + npm: string | undefined + gem: string | undefined + sourcePath: string +} + +export async function sortLibs( + libraries: Library[] +): Promise<{ sortedLibs: Library[]; totalStars: number }> { + let totalStars = 0 + const libsWithScores = await Promise.all( + libraries.map(async lib => { + const [npmStats, gemStars, githubStats, httpScore] = await Promise.all([ + lib.npm ? getNpmStats(lib.npm) : undefined, + lib.gem ? getGemStats(lib.gem) : undefined, + lib.github ? getGitHubStats(lib.github) : undefined, + lib.name ? getHttpScore(lib.name) : undefined, + ]) + + const result = { + ...lib, + downloadCount: npmStats ?? gemStars ?? 0, + stars: githubStats?.stars ?? 0, + httpScore: httpScore ?? 0, + ...githubStats, + } + totalStars += result.stars + return result + }) + ) + const sortedLibs = libsWithScores.sort((a, b) => { + let aScore = 0, + bScore = 0 + if (a.downloadCount > b.downloadCount) { + aScore += 36 + } else if (b.downloadCount > a.downloadCount) { + bScore += 36 + } + + if (a.httpScore > b.httpScore) { + aScore += 10 + } else if (b.httpScore > a.httpScore) { + bScore += 10 + } + + if ("hasCommitsInLast3Months" in a && a.hasCommitsInLast3Months) { + aScore += 28 + } + if ("hasCommitsInLast3Months" in b && b.hasCommitsInLast3Months) { + bScore += 28 + } + if (a.stars > b.stars) { + aScore += 36 + } else if (a.stars < b.stars) { + bScore += 36 + } + if (bScore > aScore) { + return 1 + } + if (bScore < aScore) { + return -1 + } + return 0 + }) + + return { sortedLibs, totalStars } +}