fix chunk timeout + update tests

This commit is contained in:
Rob Herley
2024-07-23 21:57:39 -04:00
parent 1db73622df
commit 182702d2df
3 changed files with 216 additions and 373 deletions

View File

@@ -57,3 +57,7 @@ export function getConcurrency(): number {
const concurrency = 16 * numCPUs
return concurrency > 300 ? 300 : concurrency
}
export function getUploadChunkTimeout(): number {
return 30_000
}

View File

@@ -1,7 +1,11 @@
import {BlobClient, BlockBlobUploadStreamOptions} from '@azure/storage-blob'
import {TransferProgressEvent} from '@azure/core-http'
import {ZipUploadStream} from './zip'
import {getUploadChunkSize, getConcurrency} from '../shared/config'
import {
getUploadChunkSize,
getConcurrency,
getUploadChunkTimeout
} from '../shared/config'
import * as core from '@actions/core'
import * as crypto from 'crypto'
import * as stream from 'stream'
@@ -25,29 +29,26 @@ export async function uploadZipToBlobStorage(
): Promise<BlobUploadResponse> {
let uploadByteCount = 0
let lastProgressTime = Date.now()
let timeoutId: NodeJS.Timeout | undefined
const abortController = new AbortController()
const chunkTimer = (timeout: number): NodeJS.Timeout => {
// clear the previous timeout
if (timeoutId) {
clearTimeout(timeoutId)
}
const chunkTimer = async (interval: number): Promise<void> =>
new Promise((resolve, reject) => {
const timer = setInterval(() => {
if (Date.now() - lastProgressTime > interval) {
reject(new Error('Upload progress stalled.'))
}
}, interval)
abortController.signal.addEventListener('abort', () => {
clearInterval(timer)
resolve()
})
})
timeoutId = setTimeout(() => {
const now = Date.now()
// if there's been more than 30 seconds since the
// last progress event, then we'll consider the upload stalled
if (now - lastProgressTime > timeout) {
throw new Error('Upload progress stalled.')
}
}, timeout)
return timeoutId
}
const maxConcurrency = getConcurrency()
const bufferSize = getUploadChunkSize()
const blobClient = new BlobClient(authenticatedUploadURL)
const blockBlobClient = blobClient.getBlockBlobClient()
const timeoutDuration = 300000 // 30 seconds
core.debug(
`Uploading artifact zip to blob storage with maxConcurrency: ${maxConcurrency}, bufferSize: ${bufferSize}`
@@ -56,13 +57,13 @@ export async function uploadZipToBlobStorage(
const uploadCallback = (progress: TransferProgressEvent): void => {
core.info(`Uploaded bytes ${progress.loadedBytes}`)
uploadByteCount = progress.loadedBytes
chunkTimer(timeoutDuration)
lastProgressTime = Date.now()
}
const options: BlockBlobUploadStreamOptions = {
blobHTTPHeaders: {blobContentType: 'zip'},
onProgress: uploadCallback
onProgress: uploadCallback,
abortSignal: abortController.signal
}
let sha256Hash: string | undefined = undefined
@@ -75,24 +76,22 @@ export async function uploadZipToBlobStorage(
core.info('Beginning upload of artifact content to blob storage')
try {
// Start the chunk timer
timeoutId = chunkTimer(timeoutDuration)
await blockBlobClient.uploadStream(
uploadStream,
bufferSize,
maxConcurrency,
options
)
await Promise.race([
blockBlobClient.uploadStream(
uploadStream,
bufferSize,
maxConcurrency,
options
),
chunkTimer(getUploadChunkTimeout())
])
} catch (error) {
if (NetworkError.isNetworkErrorCode(error?.code)) {
throw new NetworkError(error?.code)
}
throw error
} finally {
// clear the timeout whether or not the upload completes
if (timeoutId) {
clearTimeout(timeoutId)
}
abortController.abort()
}
core.info('Finished uploading artifact content to blob storage!')