Add backoff and jitter to deployment polling
Co-authored-by: Copilot App <[email protected]>
This commit is contained in:
Yoann Chaudet
co-authored by
Copilot App
Copilot App
parent
5e98f10ce2
commit
0143e11abb
@@ -51,7 +51,7 @@ jobs:
|
||||
| `token` | `true` | `${{ github.token }}` | The GitHub token used to create an authenticated client - Provided for you by default! |
|
||||
| `timeout` | `false` | `"600000"` | Time in milliseconds after which to timeout and cancel the deployment (default: 10 minutes) |
|
||||
| `error_count` | `false` | `"10"` | Maximum number of status report errors before cancelling a deployment (default: 10) |
|
||||
| `reporting_interval` | `false` | `"5000"` | Time in milliseconds between two deployment status reports (default: 5 seconds) |
|
||||
| `reporting_interval` | `false` | `"5000"` | Initial time in milliseconds between deployment status reports. Subsequent intervals use exponential backoff capped at 30 seconds and ±20% jitter (default: 5 seconds). |
|
||||
| `artifact_name` | `false` | `"github-pages"` | The name of the artifact to deploy |
|
||||
| `preview` | `false` | `"false"` | Is this attempting to deploy a pull request as a GitHub Pages preview site? (NOTE: This feature is only in alpha currently and is not available to the public!) |
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ inputs:
|
||||
required: false
|
||||
default: '10'
|
||||
reporting_interval:
|
||||
description: 'Time in milliseconds between two deployment status report (default: 5 seconds)'
|
||||
description: 'Initial time in milliseconds between deployment status reports; subsequent intervals use capped backoff and jitter (default: 5 seconds)'
|
||||
required: false
|
||||
default: '5000'
|
||||
artifact_name:
|
||||
|
||||
+21
-3
@@ -149920,9 +149920,17 @@ const finalErrorStatus = {
|
||||
}
|
||||
|
||||
const MAX_TIMEOUT = 600000
|
||||
const MAX_REPORTING_INTERVAL = 30000
|
||||
const REPORTING_BACKOFF_MULTIPLIER = 1.5
|
||||
const REPORTING_JITTER_FACTOR = 0.2
|
||||
const ONE_GIGABYTE = 1073741824
|
||||
const SIZE_LIMIT_DESCRIPTION = '1 GB'
|
||||
|
||||
function getJitteredInterval(interval) {
|
||||
const jitter = interval * REPORTING_JITTER_FACTOR
|
||||
return Math.round(interval - jitter + Math.random() * jitter * 2)
|
||||
}
|
||||
|
||||
class Deployment {
|
||||
constructor() {
|
||||
const context = getContext()
|
||||
@@ -150034,7 +150042,7 @@ class Deployment {
|
||||
}
|
||||
|
||||
const deploymentId = this.deploymentInfo.id || this.buildVersion
|
||||
const reportingInterval = Number(core.getInput('reporting_interval'))
|
||||
let reportingInterval = Number(core.getInput('reporting_interval'))
|
||||
const maxErrorCount = Number(core.getInput('error_count'))
|
||||
|
||||
let errorCount = 0
|
||||
@@ -150047,7 +150055,7 @@ class Deployment {
|
||||
/*eslint no-constant-condition: ["error", { "checkLoops": false }]*/
|
||||
while (true) {
|
||||
// Handle reporting interval
|
||||
await new Promise(resolve => setTimeout(resolve, reportingInterval + errorReportingInterval))
|
||||
await new Promise(resolve => setTimeout(resolve, getJitteredInterval(reportingInterval + errorReportingInterval)))
|
||||
|
||||
// Check status
|
||||
try {
|
||||
@@ -150075,6 +150083,10 @@ class Deployment {
|
||||
|
||||
// reset the error reporting interval once get the proper status back.
|
||||
errorReportingInterval = 0
|
||||
reportingInterval = Math.min(
|
||||
Math.round(reportingInterval * REPORTING_BACKOFF_MULTIPLIER),
|
||||
MAX_REPORTING_INTERVAL
|
||||
)
|
||||
} catch (error) {
|
||||
core.error(error.stack)
|
||||
|
||||
@@ -150138,7 +150150,13 @@ class Deployment {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { Deployment, MAX_TIMEOUT, ONE_GIGABYTE, SIZE_LIMIT_DESCRIPTION }
|
||||
module.exports = {
|
||||
Deployment,
|
||||
MAX_TIMEOUT,
|
||||
MAX_REPORTING_INTERVAL,
|
||||
ONE_GIGABYTE,
|
||||
SIZE_LIMIT_DESCRIPTION
|
||||
}
|
||||
|
||||
|
||||
/***/ }),
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -4,7 +4,13 @@ const nock = require('nock')
|
||||
// For mocking network calls with native Fetch (octokit)
|
||||
const { MockAgent, setGlobalDispatcher } = require('undici')
|
||||
|
||||
const { Deployment, MAX_TIMEOUT, ONE_GIGABYTE, SIZE_LIMIT_DESCRIPTION } = require('../../internal/deployment')
|
||||
const {
|
||||
Deployment,
|
||||
MAX_TIMEOUT,
|
||||
MAX_REPORTING_INTERVAL,
|
||||
ONE_GIGABYTE,
|
||||
SIZE_LIMIT_DESCRIPTION
|
||||
} = require('../../internal/deployment')
|
||||
|
||||
const fakeJwt =
|
||||
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiJiNjllMWIxOC1jOGFiLTRhZGQtOGYxOC03MzVlMzVjZGJhZjAiLCJzdWIiOiJyZXBvOnBhcGVyLXNwYS9taW55aTplbnZpcm9ubWVudDpQcm9kdWN0aW9uIiwiYXVkIjoiaHR0cHM6Ly9naXRodWIuY29tL3BhcGVyLXNwYSIsInJlZiI6InJlZnMvaGVhZHMvbWFpbiIsInNoYSI6ImEyODU1MWJmODdiZDk3NTFiMzdiMmM0YjM3M2MxZjU3NjFmYWM2MjYiLCJyZXBvc2l0b3J5IjoicGFwZXItc3BhL21pbnlpIiwicmVwb3NpdG9yeV9vd25lciI6InBhcGVyLXNwYSIsInJ1bl9pZCI6IjE1NDY0NTkzNjQiLCJydW5fbnVtYmVyIjoiMzQiLCJydW5fYXR0ZW1wdCI6IjIiLCJhY3RvciI6IllpTXlzdHkiLCJ3b3JrZmxvdyI6IkNJIiwiaGVhZF9yZWYiOiIiLCJiYXNlX3JlZiI6IiIsImV2ZW50X25hbWUiOiJwdXNoIiwicmVmX3R5cGUiOiJicmFuY2giLCJlbnZpcm9ubWVudCI6IlByb2R1Y3Rpb24iLCJqb2Jfd29ya2Zsb3dfcmVmIjoicGFwZXItc3BhL21pbnlpLy5naXRodWIvd29ya2Zsb3dzL2JsYW5rLnltbEByZWZzL2hlYWRzL21haW4iLCJpc3MiOiJodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tIiwibmJmIjoxNjM4ODI4MDI4LCJleHAiOjE2Mzg4Mjg5MjgsImlhdCI6MTYzODgyODYyOH0.1wyupfxu1HGoTyIqatYg0hIxy2-0bMO-yVlmLSMuu2w'
|
||||
@@ -35,7 +41,7 @@ describe('Deployment', () => {
|
||||
case 'token':
|
||||
return process.env.GITHUB_TOKEN
|
||||
case 'reporting_interval':
|
||||
return 50 // Lower reporting interval to speed up test
|
||||
return process.env.INPUT_REPORTING_INTERVAL || 50 // Lower reporting interval to speed up test
|
||||
default:
|
||||
return process.env[`INPUT_${param.toUpperCase()}`] || ''
|
||||
}
|
||||
@@ -608,6 +614,36 @@ describe('Deployment', () => {
|
||||
})
|
||||
|
||||
describe('#check', () => {
|
||||
const mockDeploymentStatus = (status, times = 1) => {
|
||||
mockPool
|
||||
.intercept({
|
||||
path: `/repos/${process.env.GITHUB_REPOSITORY}/pages/deployments/${process.env.GITHUB_SHA}`,
|
||||
method: 'GET'
|
||||
})
|
||||
.reply(200, { status }, { headers: { 'content-type': 'application/json' } })
|
||||
.times(times)
|
||||
}
|
||||
|
||||
const createPendingDeployment = () => {
|
||||
const deployment = new Deployment()
|
||||
deployment.deploymentInfo = {
|
||||
id: process.env.GITHUB_SHA,
|
||||
pending: true
|
||||
}
|
||||
deployment.startTime = Date.now()
|
||||
return deployment
|
||||
}
|
||||
|
||||
const runWithoutWaiting = async deployment => {
|
||||
const timeoutSpy = jest.spyOn(global, 'setTimeout').mockImplementation(resolve => {
|
||||
resolve()
|
||||
return 0
|
||||
})
|
||||
|
||||
await deployment.check()
|
||||
return timeoutSpy
|
||||
}
|
||||
|
||||
it('sets output to success when deployment is successful', async () => {
|
||||
process.env.GITHUB_SHA = 'valid-build-version'
|
||||
|
||||
@@ -990,6 +1026,81 @@ describe('Deployment', () => {
|
||||
expect(core.info).toHaveBeenLastCalledWith('Reported success!')
|
||||
twirpScope.done()
|
||||
})
|
||||
|
||||
it('backs off successful non-terminal status checks', async () => {
|
||||
process.env.GITHUB_SHA = 'valid-build-version'
|
||||
process.env.INPUT_ERROR_COUNT = '10'
|
||||
mockDeploymentStatus('deployment_in_progress', 2)
|
||||
mockDeploymentStatus('succeed')
|
||||
const randomSpy = jest.spyOn(Math, 'random').mockReturnValue(0.5)
|
||||
|
||||
const timeoutSpy = await runWithoutWaiting(createPendingDeployment())
|
||||
|
||||
expect(timeoutSpy).toHaveBeenNthCalledWith(1, expect.any(Function), 50)
|
||||
expect(timeoutSpy).toHaveBeenNthCalledWith(2, expect.any(Function), 75)
|
||||
expect(timeoutSpy).toHaveBeenNthCalledWith(3, expect.any(Function), 113)
|
||||
|
||||
timeoutSpy.mockRestore()
|
||||
randomSpy.mockRestore()
|
||||
delete process.env.INPUT_ERROR_COUNT
|
||||
})
|
||||
|
||||
it('caps the successful status check backoff', async () => {
|
||||
process.env.GITHUB_SHA = 'valid-build-version'
|
||||
process.env.INPUT_ERROR_COUNT = '10'
|
||||
process.env.INPUT_REPORTING_INTERVAL = '20000'
|
||||
mockDeploymentStatus('deployment_in_progress')
|
||||
mockDeploymentStatus('succeed')
|
||||
const randomSpy = jest.spyOn(Math, 'random').mockReturnValue(0.5)
|
||||
|
||||
const timeoutSpy = await runWithoutWaiting(createPendingDeployment())
|
||||
|
||||
expect(timeoutSpy.mock.calls.map(([, interval]) => interval)).toEqual([20000, MAX_REPORTING_INTERVAL])
|
||||
|
||||
timeoutSpy.mockRestore()
|
||||
randomSpy.mockRestore()
|
||||
delete process.env.INPUT_ERROR_COUNT
|
||||
delete process.env.INPUT_REPORTING_INTERVAL
|
||||
})
|
||||
|
||||
it('jitters status check intervals by up to twenty percent', async () => {
|
||||
process.env.GITHUB_SHA = 'valid-build-version'
|
||||
process.env.INPUT_ERROR_COUNT = '10'
|
||||
mockDeploymentStatus('deployment_in_progress')
|
||||
mockDeploymentStatus('succeed')
|
||||
const randomSpy = jest.spyOn(Math, 'random').mockReturnValueOnce(0).mockReturnValueOnce(1)
|
||||
|
||||
const timeoutSpy = await runWithoutWaiting(createPendingDeployment())
|
||||
|
||||
expect(timeoutSpy).toHaveBeenNthCalledWith(1, expect.any(Function), 40)
|
||||
expect(timeoutSpy).toHaveBeenNthCalledWith(2, expect.any(Function), 90)
|
||||
|
||||
timeoutSpy.mockRestore()
|
||||
randomSpy.mockRestore()
|
||||
delete process.env.INPUT_ERROR_COUNT
|
||||
})
|
||||
|
||||
it('keeps success backoff separate from error backoff', async () => {
|
||||
process.env.GITHUB_SHA = 'valid-build-version'
|
||||
process.env.INPUT_ERROR_COUNT = '10'
|
||||
mockPool
|
||||
.intercept({
|
||||
path: `/repos/${process.env.GITHUB_REPOSITORY}/pages/deployments/${process.env.GITHUB_SHA}`,
|
||||
method: 'GET'
|
||||
})
|
||||
.reply(500, {}, { headers: { 'content-type': 'application/json' } })
|
||||
mockDeploymentStatus('deployment_in_progress')
|
||||
mockDeploymentStatus('succeed')
|
||||
const randomSpy = jest.spyOn(Math, 'random').mockReturnValue(0.5)
|
||||
|
||||
const timeoutSpy = await runWithoutWaiting(createPendingDeployment())
|
||||
|
||||
expect(timeoutSpy.mock.calls.map(([, interval]) => interval)).toEqual([50, 51, 75])
|
||||
|
||||
timeoutSpy.mockRestore()
|
||||
randomSpy.mockRestore()
|
||||
delete process.env.INPUT_ERROR_COUNT
|
||||
})
|
||||
})
|
||||
|
||||
describe('#cancel', () => {
|
||||
|
||||
@@ -24,9 +24,17 @@ const finalErrorStatus = {
|
||||
}
|
||||
|
||||
const MAX_TIMEOUT = 600000
|
||||
const MAX_REPORTING_INTERVAL = 30000
|
||||
const REPORTING_BACKOFF_MULTIPLIER = 1.5
|
||||
const REPORTING_JITTER_FACTOR = 0.2
|
||||
const ONE_GIGABYTE = 1073741824
|
||||
const SIZE_LIMIT_DESCRIPTION = '1 GB'
|
||||
|
||||
function getJitteredInterval(interval) {
|
||||
const jitter = interval * REPORTING_JITTER_FACTOR
|
||||
return Math.round(interval - jitter + Math.random() * jitter * 2)
|
||||
}
|
||||
|
||||
class Deployment {
|
||||
constructor() {
|
||||
const context = getContext()
|
||||
@@ -138,7 +146,7 @@ class Deployment {
|
||||
}
|
||||
|
||||
const deploymentId = this.deploymentInfo.id || this.buildVersion
|
||||
const reportingInterval = Number(core.getInput('reporting_interval'))
|
||||
let reportingInterval = Number(core.getInput('reporting_interval'))
|
||||
const maxErrorCount = Number(core.getInput('error_count'))
|
||||
|
||||
let errorCount = 0
|
||||
@@ -151,7 +159,7 @@ class Deployment {
|
||||
/*eslint no-constant-condition: ["error", { "checkLoops": false }]*/
|
||||
while (true) {
|
||||
// Handle reporting interval
|
||||
await new Promise(resolve => setTimeout(resolve, reportingInterval + errorReportingInterval))
|
||||
await new Promise(resolve => setTimeout(resolve, getJitteredInterval(reportingInterval + errorReportingInterval)))
|
||||
|
||||
// Check status
|
||||
try {
|
||||
@@ -179,6 +187,10 @@ class Deployment {
|
||||
|
||||
// reset the error reporting interval once get the proper status back.
|
||||
errorReportingInterval = 0
|
||||
reportingInterval = Math.min(
|
||||
Math.round(reportingInterval * REPORTING_BACKOFF_MULTIPLIER),
|
||||
MAX_REPORTING_INTERVAL
|
||||
)
|
||||
} catch (error) {
|
||||
core.error(error.stack)
|
||||
|
||||
@@ -242,4 +254,10 @@ class Deployment {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { Deployment, MAX_TIMEOUT, ONE_GIGABYTE, SIZE_LIMIT_DESCRIPTION }
|
||||
module.exports = {
|
||||
Deployment,
|
||||
MAX_TIMEOUT,
|
||||
MAX_REPORTING_INTERVAL,
|
||||
ONE_GIGABYTE,
|
||||
SIZE_LIMIT_DESCRIPTION
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user