Use Azure storage SDK to download cache (#497)

* Adds option to download using AzCopy

* Bump version number and add release notes

* Ensure we use at least v10

* Negate env var so it disables AzCopy

* Use Azure storage SDK to download cache

* Use same level of parallelism as AzCopy

* Fix naming of variable

* React to feedback

* Bump Node types to Node 12

* Make linter happy

* Pass options into restoreCache method

* Fix tests

* Restructure files and add tests

* Add method to get the default download and upload options

* Include breaking changes in RELEASES.md

Co-authored-by: Josh Gross <joshmgross@github.com>

packages/cache/__tests__/cacheHttpClient.test.ts

@@ -1,5 +1,9 @@
-import {getCacheVersion, retry} from '../src/internal/cacheHttpClient'
+import {downloadCache, getCacheVersion} from '../src/internal/cacheHttpClient'
 import {CompressionMethod} from '../src/internal/constants'
+import * as downloadUtils from '../src/internal/downloadUtils'
+import {DownloadOptions, getDownloadOptions} from '../src/options'
+
+jest.mock('../src/internal/downloadUtils')
 
 test('getCacheVersion with one path returns version', async () => {
   const paths = ['node_modules']
@@ -35,141 +39,103 @@ test('getCacheVersion with gzip compression does not change vesion', async () =>
   )
 })
 
-interface TestResponse {
-  statusCode: number
-  result: string | null
-}
-
-async function handleResponse(
-  response: TestResponse | undefined
-): Promise<TestResponse> {
-  if (!response) {
-    // eslint-disable-next-line no-undef
-    fail('Retry method called too many times')
-  }
-
-  if (response.statusCode === 999) {
-    throw Error('Test Error')
-  } else {
-    return Promise.resolve(response)
-  }
-}
-
-async function testRetryExpectingResult(
-  responses: TestResponse[],
-  expectedResult: string | null
-): Promise<void> {
-  responses = responses.reverse() // Reverse responses since we pop from end
-
-  const actualResult = await retry(
-    'test',
-    async () => handleResponse(responses.pop()),
-    (response: TestResponse) => response.statusCode
+test('downloadCache uses http-client for non-Azure URLs', async () => {
+  const downloadCacheHttpClientMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheHttpClient'
+  )
+  const downloadCacheStorageSDKMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheStorageSDK'
   )
 
-  expect(actualResult.result).toEqual(expectedResult)
-}
+  const archiveLocation = 'http://www.actionscache.test/download'
+  const archivePath = '/foo/bar'
 
-async function testRetryExpectingError(
-  responses: TestResponse[]
-): Promise<void> {
-  responses = responses.reverse() // Reverse responses since we pop from end
+  await downloadCache(archiveLocation, archivePath)
 
-  expect(
-    retry(
-      'test',
-      async () => handleResponse(responses.pop()),
-      (response: TestResponse) => response.statusCode
-    )
-  ).rejects.toBeInstanceOf(Error)
-}
-
-test('retry works on successful response', async () => {
-  await testRetryExpectingResult(
-    [
-      {
-        statusCode: 200,
-        result: 'Ok'
-      }
-    ],
-    'Ok'
+  expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(1)
+  expect(downloadCacheHttpClientMock).toHaveBeenCalledWith(
+    archiveLocation,
+    archivePath
   )
+
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(0)
 })
 
-test('retry works after retryable status code', async () => {
-  await testRetryExpectingResult(
-    [
-      {
-        statusCode: 503,
-        result: null
-      },
-      {
-        statusCode: 200,
-        result: 'Ok'
-      }
-    ],
-    'Ok'
+test('downloadCache uses storage SDK for Azure storage URLs', async () => {
+  const downloadCacheHttpClientMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheHttpClient'
   )
-})
-
-test('retry fails after exhausting retries', async () => {
-  await testRetryExpectingError([
-    {
-      statusCode: 503,
-      result: null
-    },
-    {
-      statusCode: 503,
-      result: null
-    },
-    {
-      statusCode: 200,
-      result: 'Ok'
-    }
-  ])
-})
-
-test('retry fails after non-retryable status code', async () => {
-  await testRetryExpectingError([
-    {
-      statusCode: 500,
-      result: null
-    },
-    {
-      statusCode: 200,
-      result: 'Ok'
-    }
-  ])
-})
-
-test('retry works after error', async () => {
-  await testRetryExpectingResult(
-    [
-      {
-        statusCode: 999,
-        result: null
-      },
-      {
-        statusCode: 200,
-        result: 'Ok'
-      }
-    ],
-    'Ok'
+  const downloadCacheStorageSDKMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheStorageSDK'
   )
+
+  const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
+  const archivePath = '/foo/bar'
+
+  await downloadCache(archiveLocation, archivePath)
+
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(1)
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledWith(
+    archiveLocation,
+    archivePath,
+    getDownloadOptions()
+  )
+
+  expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(0)
 })
 
-test('retry returns after client error', async () => {
-  await testRetryExpectingResult(
-    [
-      {
-        statusCode: 400,
-        result: null
-      },
-      {
-        statusCode: 200,
-        result: 'Ok'
-      }
-    ],
-    null
+test('downloadCache passes options to download methods', async () => {
+  const downloadCacheHttpClientMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheHttpClient'
   )
+  const downloadCacheStorageSDKMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheStorageSDK'
+  )
+
+  const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
+  const archivePath = '/foo/bar'
+  const options: DownloadOptions = {downloadConcurrency: 4}
+
+  await downloadCache(archiveLocation, archivePath, options)
+
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(1)
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalled()
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledWith(
+    archiveLocation,
+    archivePath,
+    getDownloadOptions(options)
+  )
+
+  expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(0)
+})
+
+test('downloadCache uses http-client when overridden', async () => {
+  const downloadCacheHttpClientMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheHttpClient'
+  )
+  const downloadCacheStorageSDKMock = jest.spyOn(
+    downloadUtils,
+    'downloadCacheStorageSDK'
+  )
+
+  const archiveLocation = 'http://foo.blob.core.windows.net/bar/baz'
+  const archivePath = '/foo/bar'
+  const options: DownloadOptions = {useAzureSdk: false}
+
+  await downloadCache(archiveLocation, archivePath, options)
+
+  expect(downloadCacheHttpClientMock).toHaveBeenCalledTimes(1)
+  expect(downloadCacheHttpClientMock).toHaveBeenCalledWith(
+    archiveLocation,
+    archivePath
+  )
+
+  expect(downloadCacheStorageSDKMock).toHaveBeenCalledTimes(0)
 })

packages/cache/__tests__/cacheUtils.test.ts

@@ -24,3 +24,11 @@ test('unlinkFile unlinks file', async () => {
 
   await fs.rmdir(testDirectory)
 })
+
+test('assertDefined throws if undefined', () => {
+  expect(() => cacheUtils.assertDefined('test', undefined)).toThrowError()
+})
+
+test('assertDefined returns value', () => {
+  expect(cacheUtils.assertDefined('test', 5)).toBe(5)
+})

packages/cache/__tests__/options.test.ts

@@ -0,0 +1,54 @@
+import {
+  DownloadOptions,
+  UploadOptions,
+  getDownloadOptions,
+  getUploadOptions
+} from '../src/options'
+
+const useAzureSdk = true
+const downloadConcurrency = 8
+const timeoutInMs = 30000
+const uploadConcurrency = 4
+const uploadChunkSize = 32 * 1024 * 1024
+
+test('getDownloadOptions sets defaults', async () => {
+  const actualOptions = getDownloadOptions()
+
+  expect(actualOptions).toEqual({
+    useAzureSdk,
+    downloadConcurrency,
+    timeoutInMs
+  })
+})
+
+test('getDownloadOptions overrides all settings', async () => {
+  const expectedOptions: DownloadOptions = {
+    useAzureSdk: false,
+    downloadConcurrency: 14,
+    timeoutInMs: 20000
+  }
+
+  const actualOptions = getDownloadOptions(expectedOptions)
+
+  expect(actualOptions).toEqual(expectedOptions)
+})
+
+test('getUploadOptions sets defaults', async () => {
+  const actualOptions = getUploadOptions()
+
+  expect(actualOptions).toEqual({
+    uploadConcurrency,
+    uploadChunkSize
+  })
+})
+
+test('getUploadOptions overrides all settings', async () => {
+  const expectedOptions: UploadOptions = {
+    uploadConcurrency: 2,
+    uploadChunkSize: 16 * 1024 * 1024
+  }
+
+  const actualOptions = getUploadOptions(expectedOptions)
+
+  expect(actualOptions).toEqual(expectedOptions)
+})

packages/cache/__tests__/requestUtils.test.ts

@@ -0,0 +1,140 @@
+import {retry} from '../src/internal/requestUtils'
+
+interface TestResponse {
+  statusCode: number
+  result: string | null
+}
+
+async function handleResponse(
+  response: TestResponse | undefined
+): Promise<TestResponse> {
+  if (!response) {
+    // eslint-disable-next-line no-undef
+    fail('Retry method called too many times')
+  }
+
+  if (response.statusCode === 999) {
+    throw Error('Test Error')
+  } else {
+    return Promise.resolve(response)
+  }
+}
+
+async function testRetryExpectingResult(
+  responses: TestResponse[],
+  expectedResult: string | null
+): Promise<void> {
+  responses = responses.reverse() // Reverse responses since we pop from end
+
+  const actualResult = await retry(
+    'test',
+    async () => handleResponse(responses.pop()),
+    (response: TestResponse) => response.statusCode
+  )
+
+  expect(actualResult.result).toEqual(expectedResult)
+}
+
+async function testRetryExpectingError(
+  responses: TestResponse[]
+): Promise<void> {
+  responses = responses.reverse() // Reverse responses since we pop from end
+
+  expect(
+    retry(
+      'test',
+      async () => handleResponse(responses.pop()),
+      (response: TestResponse) => response.statusCode
+    )
+  ).rejects.toBeInstanceOf(Error)
+}
+
+test('retry works on successful response', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    'Ok'
+  )
+})
+
+test('retry works after retryable status code', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 503,
+        result: null
+      },
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    'Ok'
+  )
+})
+
+test('retry fails after exhausting retries', async () => {
+  await testRetryExpectingError([
+    {
+      statusCode: 503,
+      result: null
+    },
+    {
+      statusCode: 503,
+      result: null
+    },
+    {
+      statusCode: 200,
+      result: 'Ok'
+    }
+  ])
+})
+
+test('retry fails after non-retryable status code', async () => {
+  await testRetryExpectingError([
+    {
+      statusCode: 500,
+      result: null
+    },
+    {
+      statusCode: 200,
+      result: 'Ok'
+    }
+  ])
+})
+
+test('retry works after error', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 999,
+        result: null
+      },
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    'Ok'
+  )
+})
+
+test('retry returns after client error', async () => {
+  await testRetryExpectingResult(
+    [
+      {
+        statusCode: 400,
+        result: null
+      },
+      {
+        statusCode: 200,
+        result: 'Ok'
+      }
+    ],
+    null
+  )
+})

packages/cache/__tests__/restoreCache.test.ts

@@ -144,7 +144,8 @@ test('restore with gzip compressed cache found', async () => {
   expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
   expect(downloadCacheMock).toHaveBeenCalledWith(
     cacheEntry.archiveLocation,
-    archivePath
+    archivePath,
+    undefined
   )
   expect(getArchiveFileSizeIsBytesMock).toHaveBeenCalledWith(archivePath)
 
@@ -202,7 +203,8 @@ test('restore with zstd compressed cache found', async () => {
   expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
   expect(downloadCacheMock).toHaveBeenCalledWith(
     cacheEntry.archiveLocation,
-    archivePath
+    archivePath,
+    undefined
   )
   expect(getArchiveFileSizeIsBytesMock).toHaveBeenCalledWith(archivePath)
   expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~60 MB (62915000 B)`)
@@ -258,7 +260,8 @@ test('restore with cache found for restore key', async () => {
   expect(createTempDirectoryMock).toHaveBeenCalledTimes(1)
   expect(downloadCacheMock).toHaveBeenCalledWith(
     cacheEntry.archiveLocation,
-    archivePath
+    archivePath,
+    undefined
   )
   expect(getArchiveFileSizeIsBytesMock).toHaveBeenCalledWith(archivePath)
   expect(infoMock).toHaveBeenCalledWith(`Cache Size: ~0 MB (142 B)`)