Cleanup implementation and use tarballs instead of streaming zip

packages/cache/src/internal/cacheTwirpClient.ts

@@ -1,8 +1,8 @@
-import { HttpClient, HttpClientResponse, HttpCodes } from '@actions/http-client'
-import { BearerCredentialHandler } from '@actions/http-client/lib/auth'
 import { info, debug } from '@actions/core'
-import { CacheServiceClientJSON } from '../generated/results/api/v1/cache.twirp'
 import { getRuntimeToken, getCacheServiceURL } from './config'
+import { BearerCredentialHandler } from '@actions/http-client/lib/auth'
+import { HttpClient, HttpClientResponse, HttpCodes } from '@actions/http-client'
+import { CacheServiceClientJSON } from '../generated/results/api/v1/cache.twirp'
 // import {getUserAgentString} from './user-agent'
 // import {NetworkError, UsageError} from './errors'
 
@@ -16,6 +16,13 @@ interface Rpc {
     ): Promise<object | Uint8Array>
 }
 
+/**
+ * This class is a wrapper around the CacheServiceClientJSON class generated by Twirp.
+ * 
+ * It adds retry logic to the request method, which is not present in the generated client.
+ * 
+ * This class is used to interact with cache service v2.
+ */
 class CacheServiceClient implements Rpc {
     private httpClient: HttpClient
     private baseUrl: string

packages/cache/src/internal/constants.ts

@@ -35,4 +35,6 @@ export const SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\S
 
 export const TarFilename = 'cache.tar'
 
-export const ManifestFilename = 'manifest.txt'
+export const ManifestFilename = 'manifest.txt'
+
+export const CacheFileSizeLimit = 10 * Math.pow(1024, 3) // 10GiB per repository

packages/cache/src/internal/v2/download-cache.ts

@@ -1,68 +1,25 @@
 import * as core from '@actions/core'
-import * as httpClient from '@actions/http-client'
-import unzip from 'unzip-stream'
-const packageJson = require('../../../package.json')
 
-export async function StreamExtract(url: string, directory: string): Promise<void> {
-    let retryCount = 0
-    while (retryCount < 5) {
-      try {
-        await streamExtractExternal(url, directory)
-        return
-      } catch (error) {
-        retryCount++
-        core.info(
-          `Failed to download cache after ${retryCount} retries due to ${error.message}. Retrying in 5 seconds...`
-        )
-        // wait 5 seconds before retrying
-        await new Promise(resolve => setTimeout(resolve, 5000))
-      }
-    }
+import {
+  BlobClient,
+  BlockBlobClient,
+  BlobDownloadOptions,
+} from '@azure/storage-blob'
 
-    throw new Error(`Cache download failed after ${retryCount} retries.`)
-}
+export async function DownloadCacheFile(
+  signedUploadURL: string,
+  archivePath: string,
+): Promise<{}> {
+  const downloadOptions: BlobDownloadOptions = {
+    maxRetryRequests: 5,
+  }
 
-export async function streamExtractExternal(
-    url: string,
-    directory: string
-  ): Promise<void> {
-    const client = new httpClient.HttpClient(`@actions/cache-${packageJson.version}`)
-    const response = await client.get(url)
-    if (response.message.statusCode !== 200) {
-      core.info(`Failed to download cache. HTTP status code: ${response.message.statusCode}`)
-      throw new Error(
-        `Unexpected HTTP response from blob storage: ${response.message.statusCode} ${response.message.statusMessage}`
-      )
-    }
-  
-    const timeout = 30 * 1000 // 30 seconds
-  
-    return new Promise((resolve, reject) => {
-      const timerFn = (): void => {
-        response.message.destroy(
-          new Error(`Blob storage chunk did not respond in ${timeout}ms`)
-        )
-      }
-      const timer = setTimeout(timerFn, timeout)
-  
-      response.message
-        .on('data', () => {
-          timer.refresh()
-        })
-        .on('error', (error: Error) => {
-          core.info(
-            `response.message: Cache download failed: ${error.message}`
-          )
-          clearTimeout(timer)
-          reject(error)
-        })
-        .pipe(unzip.Extract({path: directory}))
-        .on('close', () => {
-          clearTimeout(timer)
-          resolve()
-        })
-        .on('error', (error: Error) => {
-          reject(error)
-        })
-    })
-  }
+  // TODO: tighten the configuration and pass the appropriate user-agent
+  const blobClient: BlobClient = new BlobClient(signedUploadURL)
+  const blockBlobClient: BlockBlobClient = blobClient.getBlockBlobClient()
+
+  core.debug(`BlobClient: ${JSON.stringify(blobClient)}`)
+  core.debug(`blockBlobClient: ${JSON.stringify(blockBlobClient)}`)
+
+  return blockBlobClient.downloadToFile(archivePath, 0, undefined, downloadOptions)
+}

packages/cache/src/internal/v2/upload-cache.ts

@@ -1,130 +1,27 @@
 import * as core from '@actions/core'
-import { CreateCacheEntryResponse } from '../../generated/results/api/v1/cache'
-import { ZipUploadStream } from '@actions/artifact/lib/internal/upload/zip'
-import { NetworkError } from '@actions/artifact/'
-import { TransferProgressEvent } from '@azure/core-http'
-import * as stream from 'stream'
-import * as crypto from 'crypto'
-
 import {
   BlobClient,
   BlockBlobClient,
-  BlockBlobUploadStreamOptions,
   BlockBlobParallelUploadOptions
 } from '@azure/storage-blob'
 
-export async function UploadCacheStream(
-  signedUploadURL: string,
-  zipUploadStream: ZipUploadStream
-): Promise<{}> {
-  let uploadByteCount = 0
-  let lastProgressTime = Date.now()
-  let timeoutId: NodeJS.Timeout | undefined
-
-  const chunkTimer = (timeout: number): NodeJS.Timeout => {
-    // clear the previous timeout
-    if (timeoutId) {
-      clearTimeout(timeoutId)
-    }
-
-    timeoutId = setTimeout(() => {
-      const now = Date.now()
-      // if there's been more than 30 seconds since the
-      // last progress event, then we'll consider the upload stalled
-      if (now - lastProgressTime > timeout) {
-        throw new Error('Upload progress stalled.')
-      }
-    }, timeout)
-    return timeoutId
-  }
-
-  const maxConcurrency = 32
-  const bufferSize = 8 * 1024 * 1024 // 8 MB Chunks
-  const blobClient = new BlobClient(signedUploadURL)
-  const blockBlobClient = blobClient.getBlockBlobClient()
-  const timeoutDuration = 300000 // 30 seconds
-
-  core.debug(
-    `Uploading cache zip to blob storage with maxConcurrency: ${maxConcurrency}, bufferSize: ${bufferSize}`
-  )
-
-  const uploadCallback = (progress: TransferProgressEvent): void => {
-    core.info(`Uploaded bytes ${progress.loadedBytes}`)
-    uploadByteCount = progress.loadedBytes
-    chunkTimer(timeoutDuration)
-    lastProgressTime = Date.now()
-  }
-
-  const options: BlockBlobUploadStreamOptions = {
-    blobHTTPHeaders: { blobContentType: 'zip' },
-    onProgress: uploadCallback
-  }
-
-  let sha256Hash: string | undefined = undefined
-  const uploadStream = new stream.PassThrough()
-  const hashStream = crypto.createHash('sha256')
-
-  zipUploadStream.pipe(uploadStream) // This stream is used for the upload
-  zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check
-
-  core.info('Beginning upload of cache to blob storage')
-  try {
-    // Start the chunk timer
-    timeoutId = chunkTimer(timeoutDuration)
-    await blockBlobClient.uploadStream(
-      uploadStream,
-      bufferSize,
-      maxConcurrency,
-      options
-    )
-  } catch (error) {
-    if (NetworkError.isNetworkErrorCode(error?.code)) {
-      throw new NetworkError(error?.code)
-    }
-    throw error
-  } finally {
-    // clear the timeout whether or not the upload completes
-    if (timeoutId) {
-      clearTimeout(timeoutId)
-    }
-  }
-
-  core.info('Finished uploading cache content to blob storage!')
-
-  hashStream.end()
-  sha256Hash = hashStream.read() as string
-  core.info(`SHA256 hash of uploaded artifact zip is ${sha256Hash}`)
-  core.info(`Uploaded: ${uploadByteCount} bytes`)
-
-  if (uploadByteCount === 0) {
-    core.error(
-      `No data was uploaded to blob storage. Reported upload byte count is 0.`
-    )
-  }
-  return {
-    uploadSize: uploadByteCount,
-    sha256Hash
-  }
-}
-
 export async function UploadCacheFile(
-  uploadURL: CreateCacheEntryResponse,
+  signedUploadURL: string,
   archivePath: string,
 ): Promise<{}> {
-  core.info(`Uploading ${archivePath} to: ${JSON.stringify(uploadURL)}`)
-
+  // TODO: tighten the configuration and pass the appropriate user-agent
   // Specify data transfer options
   const uploadOptions: BlockBlobParallelUploadOptions = {
     blockSize: 4 * 1024 * 1024, // 4 MiB max block size
-    concurrency: 2, // maximum number of parallel transfer workers
+    concurrency: 4, // maximum number of parallel transfer workers
     maxSingleShotSize: 8 * 1024 * 1024, // 8 MiB initial transfer size
   };
 
-  const blobClient: BlobClient = new BlobClient(uploadURL.signedUploadUrl)
+  const blobClient: BlobClient = new BlobClient(signedUploadURL)
   const blockBlobClient: BlockBlobClient = blobClient.getBlockBlobClient()
 
-  core.info(`BlobClient: ${JSON.stringify(blobClient)}`)
-  core.info(`blockBlobClient: ${JSON.stringify(blockBlobClient)}`)
+  core.debug(`BlobClient: ${JSON.stringify(blobClient)}`)
+  core.debug(`blockBlobClient: ${JSON.stringify(blockBlobClient)}`)
 
   return blockBlobClient.uploadFile(archivePath, uploadOptions);
 }