Tool cache install from a manifest file (#382)

packages/tool-cache/src/manifest.ts

@@ -0,0 +1,158 @@
+import * as semver from 'semver'
+import {debug} from '@actions/core'
+
+// needs to be require for core node modules to be mocked
+/* eslint @typescript-eslint/no-require-imports: 0 */
+
+import os = require('os')
+import cp = require('child_process')
+import fs = require('fs')
+
+/*
+NOTE: versions must be sorted descending by version in the manifest
+      this library short circuits on first semver spec match
+
+      platform_version is an optional filter and can be a semver spec or range
+[
+  {
+    "version": "1.2.3",
+    "stable": true,
+    "release_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6",
+    "files": [
+      {
+        "filename": "sometool-1.2.3-linux-x64.zip",
+        "arch": "x64",
+        "platform": "linux",
+        "platform_version": "18.04"
+        "download_url": "https://github.com/actions/sometool/releases/tag/1.2.3-20200402.6/sometool-1.2.3-linux-x64.zip"
+      },
+    ...
+    ]
+  },
+  ...
+]
+*/
+
+export interface IToolReleaseFile {
+  filename: string
+  // 'aix', 'darwin', 'freebsd', 'linux', 'openbsd',
+  // 'sunos', and 'win32'
+  // platform_version is an optional semver filter
+  // TODO: do we need distribution (e.g. ubuntu).
+  //       not adding yet but might need someday.
+  //       right now, 16.04 and 18.04 work
+  platform: string
+  platform_version?: string
+
+  // 'arm', 'arm64', 'ia32', 'mips', 'mipsel',
+  // 'ppc', 'ppc64', 's390', 's390x',
+  // 'x32', and 'x64'.
+  arch: string
+
+  download_url: string
+}
+
+export interface IToolRelease {
+  version: string
+  stable: boolean
+  release_url: string
+  files: IToolReleaseFile[]
+}
+
+export async function _findMatch(
+  versionSpec: string,
+  stable: boolean,
+  candidates: IToolRelease[],
+  archFilter: string
+): Promise<IToolRelease | undefined> {
+  const platFilter = os.platform()
+
+  let result: IToolRelease | undefined
+  let match: IToolRelease | undefined
+
+  let file: IToolReleaseFile | undefined
+  for (const candidate of candidates) {
+    const version = candidate.version
+
+    debug(`check ${version} satisfies ${versionSpec}`)
+    if (
+      semver.satisfies(version, versionSpec) &&
+      (!stable || candidate.stable === stable)
+    ) {
+      file = candidate.files.find(item => {
+        debug(
+          `${item.arch}===${archFilter} && ${item.platform}===${platFilter}`
+        )
+
+        let chk = item.arch === archFilter && item.platform === platFilter
+        if (chk && item.platform_version) {
+          const osVersion = module.exports._getOsVersion()
+
+          if (osVersion === item.platform_version) {
+            chk = true
+          } else {
+            chk = semver.satisfies(osVersion, item.platform_version)
+          }
+        }
+
+        return chk
+      })
+
+      if (file) {
+        debug(`matched ${candidate.version}`)
+        match = candidate
+        break
+      }
+    }
+  }
+
+  if (match && file) {
+    // clone since we're mutating the file list to be only the file that matches
+    result = Object.assign({}, match)
+    result.files = [file]
+  }
+
+  return result
+}
+
+export function _getOsVersion(): string {
+  // TODO: add windows and other linux, arm variants
+  // right now filtering on version is only an ubuntu and macos scenario for tools we build for hosted (python)
+  const plat = os.platform()
+  let version = ''
+
+  if (plat === 'darwin') {
+    version = cp.execSync('sw_vers -productVersion').toString()
+  } else if (plat === 'linux') {
+    // lsb_release process not in some containers, readfile
+    // Run cat /etc/lsb-release
+    // DISTRIB_ID=Ubuntu
+    // DISTRIB_RELEASE=18.04
+    // DISTRIB_CODENAME=bionic
+    // DISTRIB_DESCRIPTION="Ubuntu 18.04.4 LTS"
+    const lsbContents = module.exports._readLinuxVersionFile()
+    if (lsbContents) {
+      const lines = lsbContents.split('\n')
+      for (const line of lines) {
+        const parts = line.split('=')
+        if (parts.length === 2 && parts[0].trim() === 'DISTRIB_RELEASE') {
+          version = parts[1].trim()
+          break
+        }
+      }
+    }
+  }
+
+  return version
+}
+
+export function _readLinuxVersionFile(): string {
+  const lsbFile = '/etc/lsb-release'
+  let contents = ''
+
+  if (fs.existsSync(lsbFile)) {
+    contents = fs.readFileSync(lsbFile).toString()
+  }
+
+  return contents
+}

packages/tool-cache/src/tool-cache.ts

@@ -1,6 +1,7 @@
 import * as core from '@actions/core'
 import * as io from '@actions/io'
 import * as fs from 'fs'
+import * as mm from './manifest'
 import * as os from 'os'
 import * as path from 'path'
 import * as httpm from '@actions/http-client'
@@ -12,6 +13,7 @@ import {exec} from '@actions/exec/lib/exec'
 import {ExecOptions} from '@actions/exec/lib/interfaces'
 import {ok} from 'assert'
 import {RetryHelper} from './retry-helper'
+import {IHeaders} from '@actions/http-client/interfaces'
 
 export class HTTPError extends Error {
   constructor(readonly httpStatusCode: number | undefined) {
@@ -28,11 +30,13 @@ const userAgent = 'actions/tool-cache'
  *
  * @param url       url of tool to download
  * @param dest      path to download tool
+ * @param auth      authorization header
  * @returns         path to downloaded tool
  */
 export async function downloadTool(
   url: string,
-  dest?: string
+  dest?: string,
+  auth?: string
 ): Promise<string> {
   dest = dest || path.join(_getTempDirectory(), uuidV4())
   await io.mkdirP(path.dirname(dest))
@@ -51,7 +55,7 @@ export async function downloadTool(
   const retryHelper = new RetryHelper(maxAttempts, minSeconds, maxSeconds)
   return await retryHelper.execute(
     async () => {
-      return await downloadToolAttempt(url, dest || '')
+      return await downloadToolAttempt(url, dest || '', auth)
     },
     (err: Error) => {
       if (err instanceof HTTPError && err.httpStatusCode) {
@@ -71,7 +75,11 @@ export async function downloadTool(
   )
 }
 
-async function downloadToolAttempt(url: string, dest: string): Promise<string> {
+async function downloadToolAttempt(
+  url: string,
+  dest: string,
+  auth?: string
+): Promise<string> {
   if (fs.existsSync(dest)) {
     throw new Error(`Destination file path ${dest} already exists`)
   }
@@ -80,7 +88,16 @@ async function downloadToolAttempt(url: string, dest: string): Promise<string> {
   const http = new httpm.HttpClient(userAgent, [], {
     allowRetries: false
   })
-  const response: httpm.HttpClientResponse = await http.get(url)
+
+  let headers: IHeaders | undefined
+  if (auth) {
+    core.debug('set auth')
+    headers = {
+      authorization: auth
+    }
+  }
+
+  const response: httpm.HttpClientResponse = await http.get(url, headers)
   if (response.message.statusCode !== 200) {
     const err = new HTTPError(response.message.statusCode)
     core.debug(
@@ -202,7 +219,7 @@ export async function extract7z(
 export async function extractTar(
   file: string,
   dest?: string,
-  flags: string = 'xz'
+  flags: string | string[] = 'xz'
 ): Promise<string> {
   if (!file) {
     throw new Error("parameter 'file' is required")
@@ -226,7 +243,12 @@ export async function extractTar(
   const isGnuTar = versionOutput.toUpperCase().includes('GNU TAR')
 
   // Initialize args
-  const args = [flags]
+  let args: string[]
+  if (flags instanceof Array) {
+    args = flags
+  } else {
+    args = [flags]
+  }
 
   if (core.isDebug() && !flags.includes('v')) {
     args.push('-v')
@@ -463,6 +485,92 @@ export function findAllVersions(toolName: string, arch?: string): string[] {
   return versions
 }
 
+// versions-manifest
+//
+// typical pattern of a setup-* action that supports JIT would be:
+// 1. resolve semver against local cache
+//
+// 2. if no match, download
+//   a. query versions manifest to match
+//   b. if no match, fall back to source if exists (tool distribution)
+//   c. with download url, download, install and preprent path
+
+export type IToolRelease = mm.IToolRelease
+export type IToolReleaseFile = mm.IToolReleaseFile
+
+interface GitHubTreeItem {
+  path: string
+  size: string
+  url: string
+}
+
+interface GitHubTree {
+  tree: GitHubTreeItem[]
+  truncated: boolean
+}
+
+export async function getManifestFromRepo(
+  owner: string,
+  repo: string,
+  auth?: string,
+  branch = 'master'
+): Promise<IToolRelease[]> {
+  let releases: IToolRelease[] = []
+  const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/${branch}`
+
+  const http: httpm.HttpClient = new httpm.HttpClient('tool-cache')
+  const headers: IHeaders = {}
+  if (auth) {
+    core.debug('set auth')
+    headers.authorization = auth
+  }
+
+  const response = await http.getJson<GitHubTree>(treeUrl, headers)
+  if (!response.result) {
+    return releases
+  }
+
+  let manifestUrl = ''
+  for (const item of response.result.tree) {
+    if (item.path === 'versions-manifest.json') {
+      manifestUrl = item.url
+      break
+    }
+  }
+
+  headers['accept'] = 'application/vnd.github.VERSION.raw'
+  let versionsRaw = await (await http.get(manifestUrl, headers)).readBody()
+
+  if (versionsRaw) {
+    // shouldn't be needed but protects against invalid json saved with BOM
+    versionsRaw = versionsRaw.replace(/^\uFEFF/, '')
+    try {
+      releases = JSON.parse(versionsRaw)
+    } catch {
+      core.debug('Invalid json')
+    }
+  }
+
+  return releases
+}
+
+export async function findFromManifest(
+  versionSpec: string,
+  stable: boolean,
+  manifest: IToolRelease[],
+  archFilter: string = os.arch()
+): Promise<IToolRelease | undefined> {
+  // wrap the internal impl
+  const match: mm.IToolRelease | undefined = await mm._findMatch(
+    versionSpec,
+    stable,
+    manifest,
+    archFilter
+  )
+
+  return match
+}
+
 async function _createExtractFolder(dest?: string): Promise<string> {
   if (!dest) {
     // create a temp dir