@actions/artifact download artifacts (#340)

* Download Artifacts using @actions/artifact

packages/artifact/src/internal-artifact-client.ts

@@ -10,9 +10,22 @@ import {
 } from './internal-upload-http-client'
 import {UploadResponse} from './internal-upload-response'
 import {UploadOptions} from './internal-upload-options'
-import {checkArtifactName} from './internal-utils'
+import {DownloadOptions} from './internal-download-options'
+import {DownloadResponse} from './internal-download-response'
+import {checkArtifactName, createDirectoriesForArtifact} from './internal-utils'
+import {
+  listArtifacts,
+  downloadSingleArtifact,
+  getContainerItems
+} from './internal-download-http-client'
+import {getDownloadSpecification} from './internal-download-specification'
+import {
+  getWorkSpaceDirectory,
+  getDownloadArtifactConcurrency
+} from './internal-config-variables'
+import {normalize, resolve} from 'path'
 
-export {UploadResponse, UploadOptions}
+export {UploadResponse, UploadOptions, DownloadResponse, DownloadOptions}
 
 export interface ArtifactClient {
   /**
@@ -30,6 +43,25 @@ export interface ArtifactClient {
     rootDirectory: string,
     options?: UploadOptions
   ): Promise<UploadResponse>
+
+  /**
+   * Downloads a single artifact associated with a run
+   *
+   * @param name the name of the artifact being downloaded
+   * @param path optional path that denotes where the artifact will be downloaded to
+   * @param options extra options that allow for the customization of the download behavior
+   */
+  downloadArtifact(
+    name: string,
+    path?: string,
+    options?: DownloadOptions
+  ): Promise<DownloadResponse>
+
+  /**
+   * Downloads all artifacts associated with a run. Because there are multiple artifacts being downloaded, a folder will be created for each one in the specified or default directory
+   * @param path optional path that denotes where the artifacts will be downloaded to
+   */
+  downloadAllArtifacts(path?: string): Promise<DownloadResponse[]>
 }
 
 export class DefaultArtifactClient implements ArtifactClient {
@@ -100,25 +132,118 @@ export class DefaultArtifactClient implements ArtifactClient {
     return uploadResponse
   }
 
-  /*
-  Downloads a single artifact associated with a run
+  async downloadArtifact(
+    name: string,
+    path?: string | undefined,
+    options?: DownloadOptions | undefined
+  ): Promise<DownloadResponse> {
+    const artifacts = await listArtifacts()
+    if (artifacts.count === 0) {
+      throw new Error(
+        `Unable to find any artifacts for the associated workflow`
+      )
+    }
 
-  export async function downloadArtifact(
-      name: string,
-      path?: string,
-      options?: DownloadOptions
-    ): Promise<DownloadResponse> {
+    const artifactToDownload = artifacts.value.find(artifact => {
+      return artifact.name === name
+    })
+    if (!artifactToDownload) {
+      throw new Error(`Unable to find an artifact with the name: ${name}`)
+    }
 
-    TODO
+    const items = await getContainerItems(
+      artifactToDownload.name,
+      artifactToDownload.fileContainerResourceUrl
+    )
+
+    if (!path) {
+      path = getWorkSpaceDirectory()
+    }
+    path = normalize(path)
+    path = resolve(path)
+
+    // During upload, empty directories are rejected by the remote server so there should be no artifacts that consist of only empty directories
+    const downloadSpecification = getDownloadSpecification(
+      name,
+      items.value,
+      path,
+      options?.createArtifactFolder || false
+    )
+
+    if (downloadSpecification.filesToDownload.length === 0) {
+      core.info(
+        `No downloadable files were found for the artifact: ${artifactToDownload.name}`
+      )
+    } else {
+      // Create all necessary directories recursively before starting any download
+      await createDirectoriesForArtifact(
+        downloadSpecification.directoryStructure
+      )
+      await downloadSingleArtifact(downloadSpecification.filesToDownload)
+    }
+
+    return {
+      artifactName: name,
+      downloadPath: downloadSpecification.rootDownloadLocation
+    }
   }
 
-  Downloads all artifacts associated with a run. Because there are multiple artifacts being downloaded, a folder will be created for each one in the specified or default directory
+  async downloadAllArtifacts(
+    path?: string | undefined
+  ): Promise<DownloadResponse[]> {
+    const response: DownloadResponse[] = []
+    const artifacts = await listArtifacts()
+    if (artifacts.count === 0) {
+      core.info('Unable to find any artifacts for the associated workflow')
+      return response
+    }
 
-  export async function downloadAllArtifacts(
-      path?: string
-    ): Promise<DownloadResponse[]>{
+    if (!path) {
+      path = getWorkSpaceDirectory()
+    }
+    path = normalize(path)
+    path = resolve(path)
 
-      TODO
+    const ARTIFACT_CONCURRENCY = getDownloadArtifactConcurrency()
+    const parallelDownloads = [...new Array(ARTIFACT_CONCURRENCY).keys()]
+    let downloadedArtifacts = 0
+    await Promise.all(
+      parallelDownloads.map(async () => {
+        while (downloadedArtifacts < artifacts.count) {
+          const currentArtifactToDownload = artifacts.value[downloadedArtifacts]
+          downloadedArtifacts += 1
+
+          // Get container entries for the specific artifact
+          const items = await getContainerItems(
+            currentArtifactToDownload.name,
+            currentArtifactToDownload.fileContainerResourceUrl
+          )
+
+          // Promise.All is not correctly inferring that 'path' is no longer possibly undefined: https://github.com/microsoft/TypeScript/issues/34925
+          const downloadSpecification = getDownloadSpecification(
+            currentArtifactToDownload.name,
+            items.value,
+            path!, // eslint-disable-line @typescript-eslint/no-non-null-assertion
+            true
+          )
+          if (downloadSpecification.filesToDownload.length === 0) {
+            core.info(
+              `No downloadable files were found for any artifact ${currentArtifactToDownload.name}`
+            )
+          } else {
+            await createDirectoriesForArtifact(
+              downloadSpecification.directoryStructure
+            )
+            await downloadSingleArtifact(downloadSpecification.filesToDownload)
+          }
+
+          response.push({
+            artifactName: currentArtifactToDownload.name,
+            downloadPath: downloadSpecification.rootDownloadLocation
+          })
+        }
+      })
+    )
+    return response
   }
-  */
 }

packages/artifact/src/internal-config-variables.ts

@@ -10,6 +10,15 @@ export function getUploadChunkSize(): number {
   return 4 * 1024 * 1024 // 4 MB Chunks
 }
 
+export function getDownloadFileConcurrency(): number {
+  return 2
+}
+
+export function getDownloadArtifactConcurrency(): number {
+  // when downloading all artifact at once, this is number of concurrent artifacts being downloaded
+  return 1
+}
+
 export function getRuntimeToken(): string {
   const token = process.env['ACTIONS_RUNTIME_TOKEN']
   if (!token) {
@@ -33,3 +42,11 @@ export function getWorkFlowRunId(): string {
   }
   return workFlowRunId
 }
+
+export function getWorkSpaceDirectory(): string {
+  const workspaceDirectory = process.env['GITHUB_WORKSPACE']
+  if (!workspaceDirectory) {
+    throw new Error('Unable to get GITHUB_WORKSPACE env variable')
+  }
+  return workspaceDirectory
+}

packages/artifact/src/internal-contracts.ts

@@ -31,3 +31,32 @@ export interface UploadResults {
   size: number
   failedItems: string[]
 }
+
+export interface ListArtifactsResponse {
+  count: number
+  value: ArtifactResponse[]
+}
+
+export interface QueryArtifactResponse {
+  count: number
+  value: ContainerEntry[]
+}
+
+export interface ContainerEntry {
+  containerId: number
+  scopeIdentifier: string
+  path: string
+  itemType: string
+  status: string
+  fileLength?: number
+  fileEncoding?: number
+  fileType?: number
+  dateCreated: string
+  dateLastModified: string
+  createdBy: string
+  lastModifiedBy: string
+  itemLocation: string
+  contentLocation: string
+  fileId?: number
+  contentId: string
+}

packages/artifact/src/internal-download-http-client.ts

@@ -0,0 +1,133 @@
+import * as fs from 'fs'
+import {
+  createHttpClient,
+  getArtifactUrl,
+  getRequestOptions,
+  isSuccessStatusCode,
+  isRetryableStatusCode
+} from './internal-utils'
+import {URL} from 'url'
+import {
+  ListArtifactsResponse,
+  QueryArtifactResponse
+} from './internal-contracts'
+import {IHttpClientResponse} from '@actions/http-client/interfaces'
+import {HttpClient} from '@actions/http-client'
+import {DownloadItem} from './internal-download-specification'
+import {getDownloadFileConcurrency} from './internal-config-variables'
+import {warning} from '@actions/core'
+
+/**
+ * Gets a list of all artifacts that are in a specific container
+ */
+export async function listArtifacts(): Promise<ListArtifactsResponse> {
+  const artifactUrl = getArtifactUrl()
+  const client = createHttpClient()
+  const requestOptions = getRequestOptions('application/json')
+
+  const rawResponse = await client.get(artifactUrl, requestOptions)
+  const body: string = await rawResponse.readBody()
+  if (isSuccessStatusCode(rawResponse.message.statusCode) && body) {
+    return JSON.parse(body)
+  }
+  // eslint-disable-next-line no-console
+  console.log(rawResponse)
+  throw new Error(`Unable to list artifacts for the run`)
+}
+
+/**
+ * Fetches a set of container items that describe the contents of an artifact
+ * @param artifactName the name of the artifact
+ * @param containerUrl the artifact container URL for the run
+ */
+export async function getContainerItems(
+  artifactName: string,
+  containerUrl: string
+): Promise<QueryArtifactResponse> {
+  // The itemPath search parameter controls which containers will be returned
+  const resourceUrl = new URL(containerUrl)
+  resourceUrl.searchParams.append('itemPath', artifactName)
+
+  const client = createHttpClient()
+  const rawResponse = await client.get(resourceUrl.toString())
+  const body: string = await rawResponse.readBody()
+  if (isSuccessStatusCode(rawResponse.message.statusCode) && body) {
+    return JSON.parse(body)
+  }
+  // eslint-disable-next-line no-console
+  console.log(rawResponse)
+  throw new Error(`Unable to get ContainersItems from ${resourceUrl}`)
+}
+
+/**
+ * Concurrently downloads all the files that are part of an artifact
+ * @param downloadItems information about what items to download and where to save them
+ */
+export async function downloadSingleArtifact(
+  downloadItems: DownloadItem[]
+): Promise<void> {
+  const DOWNLOAD_CONCURRENCY = getDownloadFileConcurrency()
+  // Limit the number of files downloaded at a single time
+  const parallelDownloads = [...new Array(DOWNLOAD_CONCURRENCY).keys()]
+  const client = createHttpClient()
+  let downloadedFiles = 0
+  await Promise.all(
+    parallelDownloads.map(async () => {
+      while (downloadedFiles < downloadItems.length) {
+        const currentFileToDownload = downloadItems[downloadedFiles]
+        downloadedFiles += 1
+        await downloadIndividualFile(
+          client,
+          currentFileToDownload.sourceLocation,
+          currentFileToDownload.targetPath
+        )
+      }
+    })
+  )
+}
+
+/**
+ * Downloads an individual file
+ * @param client http client that will be used to make the necessary calls
+ * @param artifactLocation origin location where a file will be downloaded from
+ * @param downloadPath destination location for the file being downloaded
+ */
+export async function downloadIndividualFile(
+  client: HttpClient,
+  artifactLocation: string,
+  downloadPath: string
+): Promise<void> {
+  const stream = fs.createWriteStream(downloadPath)
+  const response = await client.get(artifactLocation)
+  if (isSuccessStatusCode(response.message.statusCode)) {
+    await pipeResponseToStream(response, stream)
+  } else if (isRetryableStatusCode(response.message.statusCode)) {
+    warning(
+      `Received http ${response.message.statusCode} during file download, will retry ${artifactLocation} after 10 seconds`
+    )
+    await new Promise(resolve => setTimeout(resolve, 10000))
+    const retryResponse = await client.get(artifactLocation)
+    if (isSuccessStatusCode(retryResponse.message.statusCode)) {
+      await pipeResponseToStream(response, stream)
+    } else {
+      // eslint-disable-next-line no-console
+      console.log(retryResponse)
+      throw new Error(`Unable to download ${artifactLocation}`)
+    }
+  } else {
+    // eslint-disable-next-line no-console
+    console.log(response)
+    throw new Error(`Unable to download ${artifactLocation}`)
+  }
+}
+
+export async function pipeResponseToStream(
+  response: IHttpClientResponse,
+  stream: NodeJS.WritableStream
+): Promise<void> {
+  return new Promise(resolve => {
+    response.message.pipe(stream).on('close', () => {
+      resolve()
+    })
+  })
+}

packages/artifact/src/internal-download-specification.ts

@@ -0,0 +1,78 @@
+import * as path from 'path'
+import {ContainerEntry} from './internal-contracts'
+
+export interface DownloadSpecification {
+  // root download location for the artifact
+  rootDownloadLocation: string
+
+  // directories that need to be created for all the items in the artifact
+  directoryStructure: string[]
+
+  // individual files that need to be downloaded as part of the artifact
+  filesToDownload: DownloadItem[]
+}
+
+export interface DownloadItem {
+  // Url that denotes where to download the item from
+  sourceLocation: string
+
+  // Information about where the file should be downloaded to
+  targetPath: string
+}
+
+/**
+ * Creates a specification for a set of files that will be downloaded
+ * @param artifactName the name of the artifact
+ * @param artifactEntries a set of container entries that describe that files that make up an artifact
+ * @param downloadPath the path where the artifact will be downloaded to
+ * @param includeRootDirectory specifies if there should be an extra directory (denoted by the artifact name) where the artifact files should be downloaded to
+ */
+export function getDownloadSpecification(
+  artifactName: string,
+  artifactEntries: ContainerEntry[],
+  downloadPath: string,
+  includeRootDirectory: boolean
+): DownloadSpecification {
+  const directories = new Set<string>()
+
+  const specifications: DownloadSpecification = {
+    rootDownloadLocation: includeRootDirectory
+      ? path.join(downloadPath, artifactName)
+      : downloadPath,
+    directoryStructure: [],
+    filesToDownload: []
+  }
+
+  for (const entry of artifactEntries) {
+    // Ignore artifacts in the container that don't begin with the same name
+    if (
+      entry.path.startsWith(`${artifactName}/`) ||
+      entry.path.startsWith(`${artifactName}\\`)
+    ) {
+      // normalize all separators to the local OS
+      const normalizedPathEntry = path.normalize(entry.path)
+      // entry.path always starts with the artifact name, if includeRootDirectory is false, remove the name from the beginning of the path
+      const filePath = path.join(
+        downloadPath,
+        includeRootDirectory
+          ? normalizedPathEntry
+          : normalizedPathEntry.replace(artifactName, '')
+      )
+
+      // Case insensitive folder structure maintained in the backend, not every folder is created so the 'folder'
+      // itemType cannot be relied upon. The file must be used to determine the directory structure
+      if (entry.itemType === 'file') {
+        // Get the directories that we need to create from the filePath for each individual file
+        directories.add(path.dirname(filePath))
+
+        specifications.filesToDownload.push({
+          sourceLocation: entry.contentLocation,
+          targetPath: filePath
+        })
+      }
+    }
+  }
+
+  specifications.directoryStructure = Array.from(directories)
+  return specifications
+}

packages/artifact/src/internal-utils.ts

@@ -1,4 +1,5 @@
 import {debug} from '@actions/core'
+import {promises as fs} from 'fs'
 import {HttpCodes, HttpClient} from '@actions/http-client'
 import {BearerCredentialHandler} from '@actions/http-client/auth'
 import {IHeaders} from '@actions/http-client/interfaces'
@@ -113,3 +114,13 @@ export function checkArtifactName(name: string): void {
     }
   }
 }
+
+export async function createDirectoriesForArtifact(
+  directories: string[]
+): Promise<void> {
+  for (const directory of directories) {
+    await fs.mkdir(directory, {
+      recursive: true
+    })
+  }
+}