feat(infra) Travis optimization (#2404)

* Fix internal fragmtn links in optimization.md

* Skip link checking or opencollective.com/webpack. Massive html response made the checker go into CPU overdrive

* Try out travis staged build

* Add proselint

* Upgrade pip before using it

* Move before-hooks around to try and make proselint install

* Try adding python as a language as well to geet an updated version

* More messing with config

* Manually handle node versioning

* Add node minor version to nvm install. Defaulted to slightly too low version

* Manually install node 8.11

* Try a matrix build to separate node and python stuff

* Add linkcheck task and try a different cahce setup

* Minor name change to test if cache works correctly across builds

* Attempt to combine matrix and staged build

* Attempt going back to staged build

* Bump travis

* Ping Travis. You alive?

* Fix broken travis.yml

* Fix wrong stage order

* Explicitly run specific lintings, exclude proselint

* Allow failures for link checker

* Change proselint cache directory. Maybe this works

* Add new script to fetch repository names that the docs should contain. Try to centralize github API usage to avoid rate limitations

* Add caching to eslint

* Remove parts of deploy.sh that are now run by travis

* Added new ./repositories to store github api results

* Replace fetch.sh with npm scripts and fetch-package-readmes.js

* Attempt to make caches more specific to the containers and stages they refer to

* Remove deprecaed fetch_packages script. Now replaced by two-step fetch-package-repos and fetch-package-readmes

* Attempt a more windows friendly build by using npm and webpack instead of shell commands

* Disable link checking for now to speed up builds

* Revert "Disable link checking for now to speed up builds"

This reverts commit 7d4bb06.

* Add dist to proselint cache so generated content also gets checked

* Remove unnessessary GITHUB_TOKEN env variable check in repo fetching script

* Revert "Add dist to proselint cache so generated content also gets checked"

This reverts commit fc7676d.

* Rework pipeline for better speed and make proselint a deployment blocker

* Rename build stage to reflect its actions

* Run content-tree only after generating all external content

* Remove link to non-existing named-modules-plugin

* Fix double slashes in edit links

* Rename stages

* Add new internal link check as a blocking check

* Fix wrong name in allowed_failures config

Author: Munter

.gitignore

@@ -8,3 +8,4 @@ src/**/_*.md
 npm-debug.log
 yarn-error.log
 package-lock.json
+.cache

.travis.yml

@@ -1,15 +1,52 @@
-branches:
-  only:
-    - master
-    - develop
+sudo: false
 language: node_js
 node_js:
   - "8"
-sudo: required
-install:
-  - yarn
-  - sudo pip install proselint
-before_script:
-  - source ./src/scripts/env.sh
-script:
-  - bash ./src/scripts/deploy.sh
+
+cache:
+  yarn: true
+  directories:
+    - node_modules
+    - dist
+    - .cache
+
+stages:
+  - Build
+  - Post-build
+
+jobs:
+  fast_finish: true
+  allow_failures:
+    - name: External Link Check
+
+  include:
+    - stage: Build
+      name: Lint and Build
+      before_install: npm install --global yarn
+      install: yarn
+      script:
+        - yarn lint:js
+        - yarn lint:markdown
+        - yarn lint:social
+        - yarn build
+        - yarn lint:links
+
+    - stage: Build
+      name: Proselint
+      language: python
+      python: 3.6
+      cache:
+        pip: true
+        directories:
+          - $HOME/.cache
+      install: pip install -r requirements.txt
+      script: cp .proselintrc ~/ && proselint src/content
+
+    - stage: Post-build
+      name: Deploy
+      if: branch = master
+      script: bash ./src/scripts/deploy.sh
+
+    - stage: Post-build
+      name: External Link Check
+      script: yarn linkcheck

package.json

@@ -26,18 +26,24 @@
   "scripts": {
     "clean": "rimraf ./dist src/content/**/_*.md src/**/_*.json",
     "start": "cross-env NODE_ENV=development webpack-dev-server --config webpack.dev.js --env.dev",
+    "update-repos": "node src/utilities/fetch-package-repos.js",
     "content": "node src/scripts/build-content-tree.js ./src/content ./src/_content.json",
     "build-test": "npm run build && http-server build/",
-    "fetch": "sh src/scripts/fetch.sh",
+    "fetch": "run-p fetch:*",
+    "fetch:readmes": "node src/utilities/fetch-package-readmes.js",
+    "fetch:supporters": "node src/utilities/fetch-supporters.js",
+    "fetch:starter-kits": "node src/utilities/fetch-starter-kits.js",
     "prebuild": "npm run clean",
-    "build": "npm run fetch && npm run content && NODE_ENV=production webpack --config webpack.prod.js && npm run sitemap && echo webpack.js.org > dist/CNAME",
+    "build": "run-s fetch content && NODE_ENV=production webpack --config webpack.prod.js",
+    "postbuild": "npm run sitemap",
     "test": "npm run lint",
     "lint": "run-s lint:*",
-    "lint:js": "eslint src --ext .js,.jsx,.md",
+    "lint:js": "eslint src --ext .js,.jsx,.md --cache true --cache-location .cache/.eslintcache",
     "lint:markdown": "markdownlint --config ./.markdownlint.json *.md ./src/content/**/*.md --ignore './src/content/**/_*.md'",
     "lint:social": "alex . -q",
     "lint:prose": "cp .proselintrc ~/ && proselint src/content",
-    "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --skip opencollective.com/webpack --todo external-redirect | tee master.tap | tap-spot",
+    "lint:links": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ -i | tee internal-links.tap | tap-spot",
+    "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --skip opencollective.com/webpack --todo external-redirect | tee external-links.tap | tap-spot",
     "sitemap": "cd dist && sitemap-static --prefix=https://webpack.js.org/ > sitemap.xml",
     "serve": "npm run build && sirv start ./dist --port 4000",
     "deploy": "gh-pages -d dist"
@@ -56,7 +62,7 @@
     ]
   },
   "devDependencies": {
-    "@octokit/rest": "^15.2.6",
+    "@octokit/rest": "^15.9.4",
     "alex": "^5.1.0",
     "autoprefixer": "^7.2.3",
     "babel-core": "^6.26.0",

repositories/README.md

@@ -0,0 +1,3 @@
+# Webpack documentation repositories
+
+The files in this directory are auto generated from `src/utils/fetch-package-repos.js` and should not be edited by hand. Any manual changes will be overwritten by the automation next time it runs.

repositories/loaders.json

@@ -0,0 +1,45 @@
+[
+  "webpack-contrib/json-loader",
+  "webpack-contrib/raw-loader",
+  "webpack-contrib/coffee-loader",
+  "webpack-contrib/css-loader",
+  "webpack-contrib/style-loader",
+  "webpack-contrib/script-loader",
+  "webpack-contrib/less-loader",
+  "webpack-contrib/bundle-loader",
+  "webpack-contrib/val-loader",
+  "webpack-contrib/file-loader",
+  "webpack-contrib/url-loader",
+  "webpack-contrib/i18n-loader",
+  "webpack-contrib/json5-loader",
+  "webpack-contrib/worker-loader",
+  "webpack-contrib/jshint-loader",
+  "webpack-contrib/imports-loader",
+  "webpack-contrib/exports-loader",
+  "webpack-contrib/mocha-loader",
+  "webpack-contrib/coverjs-loader",
+  "webpack-contrib/expose-loader",
+  "webpack-contrib/node-loader",
+  "webpack-contrib/coffee-redux-loader",
+  "webpack-contrib/transform-loader",
+  "webpack-contrib/html-loader",
+  "webpack-contrib/sass-loader",
+  "webpack-contrib/source-map-loader",
+  "webpack-contrib/react-proxy-loader",
+  "webpack-contrib/null-loader",
+  "webpack-contrib/multi-loader",
+  "webpack-contrib/istanbul-instrumenter-loader",
+  "webpack-contrib/eslint-loader",
+  "webpack-contrib/yaml-frontmatter-loader",
+  "webpack-contrib/svg-inline-loader",
+  "webpack-contrib/restyle-loader",
+  "webpack-contrib/gzip-loader",
+  "webpack-contrib/cache-loader",
+  "webpack-contrib/thread-loader",
+  "webpack-contrib/polymer-webpack-loader",
+  "webpack-contrib/workerize-loader",
+  "webpack-contrib/config-loader",
+  "babel/babel-loader",
+  "postcss/postcss-loader",
+  "peerigon/extract-loader"
+]

repositories/plugins.json

@@ -0,0 +1,15 @@
+[
+  "webpack-contrib/i18n-webpack-plugin",
+  "webpack-contrib/component-webpack-plugin",
+  "webpack-contrib/compression-webpack-plugin",
+  "webpack-contrib/extract-text-webpack-plugin",
+  "webpack-contrib/copy-webpack-plugin",
+  "webpack-contrib/npm-install-webpack-plugin",
+  "webpack-contrib/stylelint-webpack-plugin",
+  "webpack-contrib/babel-minify-webpack-plugin",
+  "webpack-contrib/uglifyjs-webpack-plugin",
+  "webpack-contrib/zopfli-webpack-plugin",
+  "webpack-contrib/closure-webpack-plugin",
+  "webpack-contrib/css-webpack-plugin",
+  "webpack-contrib/mini-css-extract-plugin"
+]

requirements.txt

@@ -0,0 +1 @@
+proselint

src/CNAME

@@ -0,0 +1 @@
+webpack.js.org

src/scripts/deploy.sh

@@ -2,26 +2,10 @@
 # see https://gist.github.com/domenic/ec8b0fc8ab45f39403dd
 set -e # Exit with nonzero exit code if anything fails
 
-SOURCE_BRANCH="master"
-
-# Pull requests and commits to other branches shouldn't try to deploy, just build to verify
-if [ "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
-    echo "Skipping deploy; just doing a build and linting links/prose/js."
-    yarn test
-    yarn build
-    exit 0
-fi
-
 # Save some useful information
 REPO=`git config remote.origin.url`
 SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:}
 
-# Run tests
-yarn test
-
-# Run our build
-yarn build
-
 # Set some git options
 git config --global user.name "Travis CI"
 git config --global user.email "ci@travis-ci.org"

src/scripts/env.sh

@@ -0,0 +1,66 @@
+const _ = require('lodash');
+const fs = require('fs');
+const path = require('path');
+const { promisify } = require('util');
+const mkdirp = promisify(require('mkdirp'));
+const request = require('request-promise');
+
+const yamlHeadmatter = require('./yaml-headmatter.js');
+const processReadme = require('./process-readme.js');
+
+const writeFile = promisify(fs.writeFile);
+const readFile = promisify(fs.readFile);
+const cwd = process.cwd();
+
+const types = ['loaders', 'plugins'];
+
+const pathMap = {
+  loaders: path.resolve(__dirname, '../content/loaders'),
+  plugins: path.resolve(__dirname, '../content/plugins')
+};
+
+async function main() {
+  for (const type of types) {
+    const outputDir = pathMap[type];
+
+    await mkdirp(outputDir);
+
+    const repos = JSON.parse(await readFile(path.resolve(__dirname, `../../repositories/${type}.json`)));
+
+    for (const repo of repos) {
+      const [org, packageName] = repo.split('/');
+      const url = `https://raw.githubusercontent.com/${repo}/master/README.md`;
+      const htmlUrl = `https://github.com/${repo}`;
+      const editUrl = `${htmlUrl}/edit/master/README.md`;
+      const fileName = path.resolve(outputDir, `_${packageName}.md`);
+
+      let title = packageName;
+
+      if (type === 'plugins') {
+        title = _.camelCase(title);
+        title = _.upperFirst(title);
+        title = title.replace(/I18N/, 'I18n');
+      }
+
+      // generate yaml matter for file
+      let headmatter = yamlHeadmatter({
+        title: title,
+        source: url,
+        edit: editUrl,
+        repo: htmlUrl
+      });
+
+      request(url)
+        .then(async content => {
+          const body = processReadme(content, { source: url });
+
+          await writeFile(fileName, headmatter + body);
+
+          console.log('Generated:', path.relative(cwd, fileName));
+        });
+    }
+  }
+}
+
+main();
+

src/scripts/fetch.sh

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const path = require('path');
+const mkdirp = require('mkdirp');
+const _ = require('lodash');
+const GithubAPI = require('@octokit/rest');
+
+const fetch = {
+  loaders: [
+    {
+      organization: 'webpack-contrib',
+      suffixes: ['-loader']
+    },
+    'babel/babel-loader',
+    'postcss/postcss-loader',
+    'peerigon/extract-loader'
+  ],
+
+  plugins: [
+    {
+      organization: 'webpack-contrib',
+      suffixes: ['-webpack-plugin', '-extract-plugin']
+    }
+  ]
+};
+
+const api = new GithubAPI();
+
+async function paginate (org) {
+  let response = await api.repos.getForOrg({ org, type: 'public', per_page: 100});
+  let {data} = response;
+
+  while (api.hasNextPage(response)) {
+    response = await api.getNextPage(response);
+    data = data.concat(response.data);
+  }
+
+  return data;
+}
+
+async function main() {
+  mkdirp.sync(path.resolve(__dirname, `../../repositories/`));
+
+  for (const [type, collection] of Object.entries(fetch)) {
+    const result = await Promise.all(collection.map(async (item) => {
+      if (typeof item === 'string') {
+        return item;
+      }
+
+      const { organization, suffixes } = item;
+
+      const repos = await paginate(organization);
+
+      return repos
+        .map(repo => repo.full_name)
+        .filter(name => suffixes.some(suffix => name.endsWith(suffix)));
+    }));
+
+    const json = JSON.stringify(_.flatten(result), undefined, 2);
+    const jsonPath = path.resolve(__dirname, `../../repositories/${type}.json`);
+
+    fs.writeFile(jsonPath, json, (err) => {
+      if (err) {
+        throw err;
+      }
+    });
+  }
+}
+
+main();