actions/toolkit
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/actions/toolkit synced 2026-04-24 20:18:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Updates to @actions/artifact package (#367)

Browse Source 
* GZip implementation

* Optimizations and cleanup

* Update tests

* More test updates

* Update packages/artifact/src/internal-utils.ts

Co-Authored-By: Josh Gross <joshmgross@github.com>

* Clarification around Upload Paths

* Refactor to make http clients classes

* GZip fixes

* Documentation around compression

* More detailed status information during large uploads

* Pretty format

* Percentage updates without rounding

* Fix edge cases with formatting numbers

* Update packages/artifact/src/internal-utils.ts

Co-Authored-By: Josh Gross <joshmgross@github.com>

* Cleanup

* Small reorg with status reporter

* PR Feedback

* Cleanup + Simplification

* Test Cleanup

* Mock updates

* More cleanup

* Format fixes

* Overhaul to the http-manager

* Fix tests

* Promisify stats

* Documentation around implementation

* Improvements to documentation

* PR Feedback

* Remove Downloading multiple artifacts concurrently

Co-authored-by: Josh Gross <joshmgross@github.com>
This commit is contained in:
Konrad Pabjan
2020-03-12 14:50:27 +01:00
committed by GitHub
parent 5859d7172e
commit dffb5572a9
30 changed files with 1252 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
packages/artifact/src/internal/download-http-client.ts Normal file
View File

@@ -0,0 +1,189 @@
import * as fs from 'fs'
import * as zlib from 'zlib'
import {
getArtifactUrl,
getRequestOptions,
isSuccessStatusCode,
isRetryableStatusCode,
createHttpClient
} from './utils'
import {URL} from 'url'
import {ListArtifactsResponse, QueryArtifactResponse} from './contracts'
import {IHttpClientResponse} from '@actions/http-client/interfaces'
import {HttpManager} from './http-manager'
import {DownloadItem} from './download-specification'
import {
getDownloadFileConcurrency,
getRetryWaitTimeInMilliseconds
} from './config-variables'
import {warning} from '@actions/core'
import {IncomingHttpHeaders} from 'http'
export class DownloadHttpClient {
// http manager is used for concurrent connection when downloading mulitple files at once
private downloadHttpManager: HttpManager
constructor() {
this.downloadHttpManager = new HttpManager(getDownloadFileConcurrency())
}
/**
* Gets a list of all artifacts that are in a specific container
*/
async listArtifacts(): Promise<ListArtifactsResponse> {
const artifactUrl = getArtifactUrl()
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediatly
const client = this.downloadHttpManager.getClient(0)
const requestOptions = getRequestOptions('application/json')
const rawResponse = await client.get(artifactUrl, requestOptions)
const body: string = await rawResponse.readBody()
if (isSuccessStatusCode(rawResponse.message.statusCode) && body) {
return JSON.parse(body)
}
// eslint-disable-next-line no-console
console.log(rawResponse)
throw new Error(`Unable to list artifacts for the run`)
}
/**
* Fetches a set of container items that describe the contents of an artifact
* @param artifactName the name of the artifact
* @param containerUrl the artifact container URL for the run
*/
async getContainerItems(
artifactName: string,
containerUrl: string
): Promise<QueryArtifactResponse> {
// the itemPath search parameter controls which containers will be returned
const resourceUrl = new URL(containerUrl)
resourceUrl.searchParams.append('itemPath', artifactName)
// no concurrent calls so a single httpClient without the http-manager is sufficient
const client = createHttpClient()
// no keep-alive header, client disposal is not necessary
const requestOptions = getRequestOptions('application/json')
const rawResponse = await client.get(resourceUrl.toString(), requestOptions)
const body: string = await rawResponse.readBody()
if (isSuccessStatusCode(rawResponse.message.statusCode) && body) {
return JSON.parse(body)
}
// eslint-disable-next-line no-console
console.log(rawResponse)
throw new Error(`Unable to get ContainersItems from ${resourceUrl}`)
}
/**
* Concurrently downloads all the files that are part of an artifact
* @param downloadItems information about what items to download and where to save them
*/
async downloadSingleArtifact(downloadItems: DownloadItem[]): Promise<void> {
const DOWNLOAD_CONCURRENCY = getDownloadFileConcurrency()
// limit the number of files downloaded at a single time
const parallelDownloads = [...new Array(DOWNLOAD_CONCURRENCY).keys()]
let downloadedFiles = 0
await Promise.all(
parallelDownloads.map(async index => {
while (downloadedFiles < downloadItems.length) {
const currentFileToDownload = downloadItems[downloadedFiles]
downloadedFiles += 1
await this.downloadIndividualFile(
index,
currentFileToDownload.sourceLocation,
currentFileToDownload.targetPath
)
}
})
)
// done downloading, safety dispose all connections
this.downloadHttpManager.disposeAndReplaceAllClients()
}
/**
* Downloads an individual file
* @param httpClientIndex the index of the http client that is used to make all of the calls
* @param artifactLocation origin location where a file will be downloaded from
* @param downloadPath destination location for the file being downloaded
*/
private async downloadIndividualFile(
httpClientIndex: number,
artifactLocation: string,
downloadPath: string
): Promise<void> {
const stream = fs.createWriteStream(downloadPath)
const client = this.downloadHttpManager.getClient(httpClientIndex)
const requestOptions = getRequestOptions('application/octet-stream', true)
const response = await client.get(artifactLocation, requestOptions)
// check the response headers to determine if the file was compressed using gzip
const isGzip = (headers: IncomingHttpHeaders): boolean => {
return (
'content-encoding' in headers && headers['content-encoding'] === 'gzip'
)
}
if (isSuccessStatusCode(response.message.statusCode)) {
await this.pipeResponseToStream(
response,
stream,
isGzip(response.message.headers)
)
} else if (isRetryableStatusCode(response.message.statusCode)) {
warning(
`Received http ${response.message.statusCode} during file download, will retry ${artifactLocation} after 10 seconds`
)
// if an error is encountered, dispose of the http connection, and create a new one
this.downloadHttpManager.disposeAndReplaceClient(httpClientIndex)
await new Promise(resolve =>
setTimeout(resolve, getRetryWaitTimeInMilliseconds())
)
const retryResponse = await client.get(artifactLocation)
if (isSuccessStatusCode(retryResponse.message.statusCode)) {
await this.pipeResponseToStream(
response,
stream,
isGzip(response.message.headers)
)
} else {
// eslint-disable-next-line no-console
console.log(retryResponse)
throw new Error(`Unable to download ${artifactLocation}`)
}
} else {
// eslint-disable-next-line no-console
console.log(response)
throw new Error(`Unable to download ${artifactLocation}`)
}
}
/**
* Pipes the response from downloading an individual file to the appropriate stream
* @param response the http response recieved when downloading a file
* @param stream the stream where the file should be written to
* @param isGzip does the response need to be be uncompressed
*/
private async pipeResponseToStream(
response: IHttpClientResponse,
stream: NodeJS.WritableStream,
isGzip: boolean
): Promise<void> {
return new Promise(resolve => {
if (isGzip) {
// pipe the response into gunzip to decompress
const gunzip = zlib.createGunzip()
response.message
.pipe(gunzip)
.pipe(stream)
.on('close', () => {
resolve()
})
} else {
response.message.pipe(stream).on('close', () => {
resolve()
})
}
})
}
}