Iterating on Crowdin scripts, Link, pages, etc

Author: Brian Vaughn

.gitignore

@@ -3,3 +3,4 @@
 .idea
 node_modules
 public
+yarn-error.log

crowdin/.gitignore

@@ -1,2 +1,2 @@
-__translations/
+__exports/
 translations/

crowdin/README.md

@@ -0,0 +1,48 @@
+## How does this work?
+
+### Downloading content from Crowdin
+
+This subdirectory contains some JavaScript files as well as a symlink for the default language (English) that points to [the `content` directory](https://github.com/reactjs/reactjs.org/tree/master/content):
+```sh
+.
+└── crowdin
+   ├── config.js    # Crowdin configuration settings
+   ├── download.js  # Node Download script
+   └── translations
+       └── en-US -> ../../content
+```
+
+Translations can be downloaded locally with `yarn crowdin:download`. This uses the Crowdin API to download data into an `__exports` subdirectory:
+```sh
+.
+└── crowdin
+   ├── config.js    # Crowdin configuration settings
+   ├── download.js  # Node Download script
+   ├── translations
+   │    └── en-US -> ../../content
+   └── __exports
+        └── ...     # Downloaded translations go here
+```
+
+Next the task identifies which languages have been translated past a certain threshold (specified by `crowdin/config.js`). For these languages, the script creates symlinks in the `translations` subdirectory:
+```sh
+.
+└── crowdin
+   ├── config.js    # Crowdin configuration settings
+   ├── download.js  # Node Download script
+   ├── translations
+   │    ├── en-US -> ../../content
+   │    ├── es-ES -> ../__exports/.../es-ES
+   │    ├── zh-CN -> ../__exports/.../zh-CN
+   │    └── ...     # Other symlinks go here
+   └── __exports
+        └── ...     # Downloaded translations go here
+```
+
+### Gatsby integration
+
+A new (local) `gatsby-plugin-crowdin` plugin has been created that knows how to create localized links to certain sections of the website. **For now, only content from [the `content/docs` directory](https://github.com/reactjs/reactjs.org/tree/master/content/docs) is localized. All other sections/pages remain English only.**
+
+The `gatsby-source-filesystem` plugin has also been reconfigured to read all content from the `crowdin/translations/*` (symlinked) directories rather than `content`. This way it consumes translated content when available. (Crowdin provides default language content for sections that have not yet been translated for any given locale.)
+
+Because of the default symlink (`crowdin/translations/en-US` -> `content`) Gatsby will serve English content when run locally, even if the Crowdin script has not been run. This should enable fast iteration and creation of new content.

crowdin/config.js

@@ -1,6 +1,9 @@
+const path = require('path');
+
+// Also relates to the crowdin.yaml file in the root directory
 module.exports = {
   key: process.env.CROWDIN_API_KEY,
   url: 'https://api.crowdin.com/api/project/react',
   threshold: 50,
-  downloadedRootDirectory: 'test-17',
+  downloadedRootDirectory: path.join('test-17', 'docs'),
 };

crowdin/download.js

@@ -4,14 +4,12 @@ const path = require('path');
 const {symlink, lstatSync, readdirSync} = require('fs');
 
 const SYMLINKED_TRANSLATIONS_PATH = path.resolve(__dirname, 'translations');
-const DOWNLOADED_TRANSLATIONS_PATH = path.resolve(__dirname, '__translations');
+const DOWNLOADED_TRANSLATIONS_PATH = path.resolve(__dirname, '__exports');
 
 // Path to the "docs" folder within the downloaded Crowdin translations bundle.
 const downloadedDocsPath = path.resolve(
-  __dirname,
-  '__translations',
+  DOWNLOADED_TRANSLATIONS_PATH,
   config.downloadedRootDirectory,
-  'docs',
 );
 
 // Sanity check (local) Crowdin config file for expected values.

package.json

@@ -80,7 +80,7 @@
     "check-all": "npm-run-all prettier --parallel lint flow",
     "ci-check": "npm-run-all prettier:diff --parallel lint flow",
     "crowdin:download": "node ./crowdin/download",
-    "dev": "gatsby develop -H 0.0.0.0",
+    "dev": "yarn start",
     "flow": "flow",
     "format:source": "prettier --config .prettierrc --write \"{gatsby-*.js,{flow-typed,plugins,src}/**/*.js}\"",
     "format:examples": "prettier --config .prettierrc.examples --write \"examples/**/*.js\"",
@@ -92,7 +92,8 @@
     "prettier:diff": "yarn nit:source && yarn nit:examples",
     "reset": "yarn reset:cache && yarn reset:translations",
     "reset:cache": "rimraf ./.cache",
-    "reset:translations": "rimraf ./crowdin/__translations && find crowdin/translations -type l -not -name '*en-US' -delete"
+    "reset:translations": "rimraf ./crowdin/__translations && find crowdin/translations -type l -not -name '*en-US' -delete",
+    "start": "gatsby develop -H 0.0.0.0"
   },
   "devDependencies": {
     "eslint-config-prettier": "^2.6.0",

plugins/gatsby-plugin-crowdin/Link.js

@@ -12,10 +12,10 @@ import React from 'react';
 import {getLanguageCodeFromPath} from './utils';
 
 // TODO THis is a hack :( Pass this down via context or some other way?
-const DEFAULT_LANGUAGE = 'en';
+const DEFAULT_LANGUAGE = 'en-US';
 
-const DecoratedLink = ({location, to, ...rest}, ...other) => {
-  if (to.startsWith('/')) {
+const DecoratedLink = ({isLocalized, location, to, ...rest}, ...other) => {
+  if (isLocalized !== false && to.startsWith('/')) {
     const languageCode =
       getLanguageCodeFromPath(location.pathname.substr(1)) || DEFAULT_LANGUAGE;
 
@@ -29,6 +29,7 @@ const DecoratedLink = ({location, to, ...rest}, ...other) => {
 };
 
 DecoratedLink.propTypes = {
+  isLocalized: PropTypes.bool,
   location: PropTypes.object.isRequired,
   to: PropTypes.string.isRequired,
 };

src/pages/blog/all.html.js

@@ -5,7 +5,7 @@
  * @flow
  */
 
-import Link from 'gatsby-link';
+import Link from 'gatsby-plugin-crowdin/Link';
 import Container from 'components/Container';
 import Header from 'components/Header';
 import TitleAndMetaTags from 'components/TitleAndMetaTags';
@@ -19,9 +19,10 @@ import type {allMarkdownRemarkData} from 'types';
 
 type Props = {
   data: allMarkdownRemarkData,
+  location: Location,
 };
 
-const AllBlogPosts = ({data}: Props) => (
+const AllBlogPosts = ({data, location}: Props) => (
   <Container>
     <div css={sharedStyles.articleLayout.container}>
       <div css={sharedStyles.articleLayout.content}>
@@ -53,7 +54,7 @@ const AllBlogPosts = ({data}: Props) => (
                   width: '33.33%',
                 },
               }}
-              key={node.fields.slug}>
+              key={node.fields.id}>
               <h2
                 css={{
                   fontSize: 24,
@@ -68,7 +69,8 @@ const AllBlogPosts = ({data}: Props) => (
                       borderBottomColor: colors.black,
                     },
                   }}
-                  key={node.fields.slug}
+                  key={node.fields.id}
+                  location={location}
                   to={node.fields.slug}>
                   {node.frontmatter.title}
                 </Link>
@@ -116,6 +118,7 @@ export const pageQuery = graphql`
           }
           fields {
             date(formatString: "MMMM DD, YYYY")
+            id
             slug
           }
         }

src/templates/blog.js

@@ -14,10 +14,11 @@ const toSectionList = allMarkdownRemark => [
     title: 'Recent Posts',
     items: allMarkdownRemark.edges
       .map(({node}) => ({
-        id: node.fields.id,
+        id: node.fields.slug,
         title: node.frontmatter.title,
       }))
       .concat({
+        isLocalized: false,
         id: '/blog/all.html',
         title: 'All posts ...',
       }),
@@ -76,6 +77,7 @@ export const pageQuery = graphql`
           }
           fields {
             id
+            slug
           }
         }
       }

src/templates/components/NavigationFooter/NavigationFooter.js

@@ -11,7 +11,7 @@ import PropTypes from 'prop-types';
 import React from 'react';
 import {colors, fonts, media} from 'theme';
 
-const NavigationFooter = ({next, prev, location}) => {
+const NavigationFooter = ({location, next, prev}) => {
   return (
     <div
       css={{

src/templates/components/Sidebar/Section.js

@@ -79,6 +79,7 @@ class Section extends React.Component {
                 isActive: isScrollSync
                   ? activeItemId === item.id
                   : isItemActive(location, item),
+                isLocalized: item.isLocalized,
                 item,
                 location,
                 onLinkClick,
@@ -93,6 +94,7 @@ class Section extends React.Component {
                         isActive: isScrollSync
                           ? activeItemId === subitem.id
                           : isItemActive(location, subitem),
+                        isLocalized: subitem.isLocalized,
                         item: subitem,
                         location,
                         onLinkClick,

src/utils/createLink.js

@@ -15,20 +15,23 @@ import type {Node} from 'react';
 
 type CreateLinkBaseProps = {
   isActive: boolean,
+  isLocalized: ?boolean,
   item: Object,
   location: string,
   section: Object,
 };
 
 const createLinkBlog = ({
   isActive,
+  isLocalized,
   item,
   location,
   section,
 }: CreateLinkBaseProps): Node => {
   return (
     <Link
       css={[linkCss, isActive && activeLinkCss]}
+      isLocalized={isLocalized}
       location={location}
       to={item.id}>
       {isActive && <span css={activeLinkBefore} />}
@@ -39,6 +42,7 @@ const createLinkBlog = ({
 
 const createLinkCommunity = ({
   isActive,
+  isLocalized,
   item,
   location,
   section,
@@ -60,6 +64,7 @@ const createLinkCommunity = ({
   }
   return createLinkDocs({
     isActive,
+    isLocalized,
     item,
     location,
     section,
@@ -68,13 +73,15 @@ const createLinkCommunity = ({
 
 const createLinkDocs = ({
   isActive,
+  isLocalized,
   item,
   location,
   section,
 }: CreateLinkBaseProps): Node => {
   return (
     <Link
       css={[linkCss, isActive && activeLinkCss]}
+      isLocalized={isLocalized}
       location={location}
       to={slugify(item.id, section.directory)}>
       {isActive && <span css={activeLinkBefore} />}
@@ -89,6 +96,7 @@ type CreateLinkTutorialProps = {
 
 const createLinkTutorial = ({
   isActive,
+  isLocalized,
   item,
   location,
   onLinkClick,
@@ -97,6 +105,7 @@ const createLinkTutorial = ({
   return (
     <Link
       css={[linkCss, isActive && activeLinkCss]}
+      isLocalized={isLocalized}
       location={location}
       onClick={onLinkClick}
       to={item.href}>