Skip to content

Commit fc61c88

Browse files
GatsbyJS Botaxe312ger
GatsbyJS Bot
and
axe312ger
authored
feat(gatsby-source-contentful): Increase Contentful sync by up to 10x (#30422) (#30643)
Co-authored-by: Ward Peeters <ward@coding-tech.com> (cherry picked from commit b9791fe) Co-authored-by: Benedikt Rötsch <axe312ger@users.noreply.github.com>
1 parent b3315a0 commit fc61c88

File tree

5 files changed

+189
-16
lines changed

5 files changed

+189
-16
lines changed

‎packages/gatsby-source-contentful/src/__tests__/fetch-backoff.js

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/**
2+
* @jest-environment node
3+
*/
4+
5+
import nock from "nock"
6+
import fetchData from "../fetch"
7+
import { createPluginConfig } from "../plugin-options"
8+
9+
const host = `localhost`
10+
const options = {
11+
spaceId: `12345`,
12+
accessToken: `67890`,
13+
host,
14+
contentfulClientConfig: {
15+
retryLimit: 2,
16+
},
17+
}
18+
19+
const baseURI = `https://${host}`
20+
21+
const start = jest.fn()
22+
const end = jest.fn()
23+
const mockActivity = {
24+
start,
25+
end,
26+
tick: jest.fn(),
27+
done: end,
28+
}
29+
30+
const reporter = {
31+
info: jest.fn(),
32+
verbose: jest.fn(),
33+
warn: jest.fn(),
34+
panic: jest.fn(e => {
35+
throw e
36+
}),
37+
activityTimer: jest.fn(() => mockActivity),
38+
createProgress: jest.fn(() => mockActivity),
39+
}
40+
41+
const pluginConfig = createPluginConfig(options)
42+
43+
describe(`fetch-backoff`, () => {
44+
afterEach(() => {
45+
nock.cleanAll()
46+
reporter.verbose.mockClear()
47+
reporter.panic.mockClear()
48+
reporter.warn.mockClear()
49+
})
50+
51+
test(`backoffs page limit when limit is reached`, async () => {
52+
jest.setTimeout(30000)
53+
const scope = nock(baseURI)
54+
// Space
55+
.get(`/spaces/${options.spaceId}/`)
56+
.reply(200, { items: [] })
57+
// Locales
58+
.get(`/spaces/${options.spaceId}/environments/master/locales`)
59+
.reply(200, { items: [{ code: `en`, default: true }] })
60+
// Sync with 1000 (to much)
61+
.get(
62+
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=1000`
63+
)
64+
.times(1)
65+
.reply(400, {
66+
sys: { type: `Error`, id: `BadRequest` },
67+
message: `Response size too big. Maximum allowed response size: 512000B.`,
68+
requestId: `12345`,
69+
})
70+
// Sync with 666 (still to much)
71+
.get(
72+
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=666`
73+
)
74+
.times(1)
75+
.reply(400, {
76+
sys: { type: `Error`, id: `BadRequest` },
77+
message: `Response size too big. Maximum allowed response size: 512000B.`,
78+
requestId: `12345`,
79+
})
80+
// Sync with 444
81+
.get(
82+
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=444`
83+
)
84+
.reply(200, { items: [] })
85+
// Content types
86+
.get(
87+
`/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=1000&order=sys.createdAt`
88+
)
89+
.reply(200, { items: [] })
90+
91+
await fetchData({ pluginConfig, reporter })
92+
93+
expect(reporter.panic).not.toBeCalled()
94+
expect(reporter.warn.mock.calls).toMatchInlineSnapshot(`
95+
Array [
96+
Array [
97+
"The sync with Contentful failed using pageLimit 1000 as the reponse size limit of the API is exceeded.
98+
99+
Retrying sync with pageLimit of 666",
100+
],
101+
Array [
102+
"The sync with Contentful failed using pageLimit 666 as the reponse size limit of the API is exceeded.
103+
104+
Retrying sync with pageLimit of 444",
105+
],
106+
Array [
107+
"We recommend you to set your pageLimit in gatsby-config.js to 444 to avoid failed synchronizations.",
108+
],
109+
]
110+
`)
111+
expect(scope.isDone()).toBeTruthy()
112+
})
113+
114+
test(`does not backoff page limit when limit is not reached`, async () => {
115+
jest.setTimeout(30000)
116+
const scope = nock(baseURI)
117+
// Space
118+
.get(`/spaces/${options.spaceId}/`)
119+
.reply(200, { items: [] })
120+
// Locales
121+
.get(`/spaces/${options.spaceId}/environments/master/locales`)
122+
.reply(200, { items: [{ code: `en`, default: true }] })
123+
// Sync with 1000 (no limit exceeded)
124+
.get(
125+
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=1000`
126+
)
127+
.reply(200, { items: [] })
128+
// Content types
129+
.get(
130+
`/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=1000&order=sys.createdAt`
131+
)
132+
.reply(200, { items: [] })
133+
134+
await fetchData({ pluginConfig, reporter })
135+
136+
expect(reporter.panic).not.toBeCalled()
137+
expect(reporter.warn).not.toBeCalled()
138+
expect(scope.isDone()).toBeTruthy()
139+
})
140+
})

‎packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,17 +57,17 @@ describe(`fetch-retry`, () => {
5757
.reply(200, { items: [{ code: `en`, default: true }] })
5858
// Sync
5959
.get(
60-
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=100`
60+
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=1000`
6161
)
6262
.times(1)
6363
.replyWithError({ code: `ETIMEDOUT` })
6464
.get(
65-
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=100`
65+
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=1000`
6666
)
6767
.reply(200, { items: [] })
6868
// Content types
6969
.get(
70-
`/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=100&order=sys.createdAt`
70+
`/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=1000&order=sys.createdAt`
7171
)
7272
.reply(200, { items: [] })
7373

@@ -90,7 +90,7 @@ describe(`fetch-retry`, () => {
9090
.reply(200, { items: [{ code: `en`, default: true }] })
9191
// Sync
9292
.get(
93-
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=100`
93+
`/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=1000`
9494
)
9595
.times(3)
9696
.reply(

‎packages/gatsby-source-contentful/src/__tests__/fetch.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ it(`calls contentful.getContentTypes with default page limit`, async () => {
157157

158158
expect(reporter.panic).not.toBeCalled()
159159
expect(mockClient.getContentTypes).toHaveBeenCalledWith({
160-
limit: 100,
160+
limit: 1000,
161161
order: `sys.createdAt`,
162162
skip: 0,
163163
})

‎packages/gatsby-source-contentful/src/fetch.js

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -196,22 +196,55 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`,
196196
}
197197

198198
let currentSyncData
199-
const basicSyncConfig = {
200-
limit: pageLimit,
201-
resolveLinks: false,
202-
}
199+
let currentPageLimit = pageLimit
200+
let lastCurrentPageLimit
201+
let syncSuccess = false
203202
try {
204203
syncProgress = reporter.createProgress(
205204
`Contentful: ${syncToken ? `Sync changed items` : `Sync all items`}`,
206-
pageLimit,
205+
currentPageLimit,
207206
0
208207
)
209208
syncProgress.start()
210-
reporter.verbose(`Contentful: Sync ${pageLimit} items per page.`)
211-
let query = syncToken
212-
? { nextSyncToken: syncToken, ...basicSyncConfig }
213-
: { initial: true, ...basicSyncConfig }
214-
currentSyncData = await client.sync(query)
209+
reporter.verbose(`Contentful: Sync ${currentPageLimit} items per page.`)
210+
211+
while (!syncSuccess) {
212+
try {
213+
const basicSyncConfig = {
214+
limit: currentPageLimit,
215+
resolveLinks: false,
216+
}
217+
const query = syncToken
218+
? { nextSyncToken: syncToken, ...basicSyncConfig }
219+
: { initial: true, ...basicSyncConfig }
220+
currentSyncData = await client.sync(query)
221+
syncSuccess = true
222+
} catch (e) {
223+
// Back off page limit if responses content length exceeds Contentfuls limits.
224+
if (
225+
e.response?.data?.message.includes(`Response size too big`) &&
226+
currentPageLimit > 1
227+
) {
228+
lastCurrentPageLimit = currentPageLimit
229+
// Reduce page limit by a arbitrary 1/3 of the current limit to ensure
230+
// the new and bigger entries are synced without exceeding the reponse size limit
231+
currentPageLimit = Math.floor((currentPageLimit / 3) * 2) || 1
232+
reporter.warn(
233+
[
234+
`The sync with Contentful failed using pageLimit ${lastCurrentPageLimit} as the reponse size limit of the API is exceeded.`,
235+
`Retrying sync with pageLimit of ${currentPageLimit}`,
236+
].join(`\n\n`)
237+
)
238+
continue
239+
}
240+
throw e
241+
}
242+
if (currentPageLimit !== pageLimit) {
243+
reporter.warn(
244+
`We recommend you to set your pageLimit in gatsby-config.js to ${currentPageLimit} to avoid failed synchronizations.`
245+
)
246+
}
247+
}
215248
} catch (e) {
216249
reporter.panic({
217250
id: CODES.SyncError,

‎packages/gatsby-source-contentful/src/plugin-options.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ const chalk = require(`chalk`)
22

33
const _ = require(`lodash`)
44

5-
const DEFAULT_PAGE_LIMIT = 100
5+
const DEFAULT_PAGE_LIMIT = 1000
66

77
const defaultOptions = {
88
host: `cdn.contentful.com`,

0 commit comments

Comments
 (0)