Merge pull request #1780 from rbenv/actions-mirror

    - uses: actions/checkout@v2

      with:

        fetch-depth: 0

    - run: git clone --depth 1 https://github.com/sstephenson/bats.git

    - run: PATH="./bats/bin:$PATH" script/test

    - name: Install bats

      run: git clone --depth 1 https://github.com/sstephenson/bats.git

    - name: Run tests

      run: PATH="./bats/bin:$PATH" script/test

    - name: Verify download URL checksums

      if: github.event_name == 'pull_request'

      run: ./script/mirror verify "$COMMIT_RANGE"

      env:

        COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}

        COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..

name: Mirror

on:

  push:

    branches:

    - master

  workflow_dispatch:

    inputs:

      beforeRef:

        description: Git commit reference to start comparing from

        required: true

jobs:

  test:

    runs-on: ubuntu-latest

    steps:

    - uses: actions/checkout@v2

      with:

        fetch-depth: 0

    - name: Update download mirror

      run: script/mirror update "${BEFORE_REF}.."

      env:

        BEFORE_REF: ${{ github.event.before }}${{ github.event.inputs.beforeRef }}

        AMAZON_S3_BUCKET: ruby-build-mirror

        AWS_ACCESS_KEY_ID: AKIAJKAUQVHU6X4CODDQ

        AWS_SECRET_ACCESS_KEY: ${{ secrets.MIRROR_UPLOAD_SECRET }}

sudo: false

install: git clone --depth 1 https://github.com/sstephenson/bats.git

script: PATH="./bats/bin:$PATH" script/travis

language: c

notifications:

  email:

    on_success: never

env:

  global:

    - AMAZON_S3_BUCKET=ruby-build-mirror

    - AMAZON_ACCESS_KEY_ID=AKIAJKAUQVHU6X4CODDQ

    - secure: LTSvDP2o72nbECDwWsfwnsiETF4VpqrYN3y/ve68AZIMzfNWDB5vhqzMLU1ltFnSNxd71gTCGX2OEcsxdrfnG+Msu52v8FtJ7lz/b9xn83gGYrGnmEMzARtd1fnuzlWQh/1eNL9jrNl8FDhgjoTqKl2gF6fZBsQxcHRnvRSXcqE=

}

compute_sha2() {

  local output="$(openssl dgst -sha256)"

  echo "${output##* }" | tr '[A-Z]' '[a-z]'

  local output

  output="$(openssl dgst -sha256)"

  tr '[:upper:]' '[:lower:]' <<<"${output##* }"

}

download_package() {

}

potentially_new_packages() {

  local files="$(changed_files "$1" -- ./share/ruby-build)"

  [ -n "$files" ] && extract_urls $files

  local head="${1#*..}"

  local files

  IFS=$'\n' read -d '' -r -a files < <(changed_files "$1" -- ./share/ruby-build)

  [ ${#files[@]} -gt 0 ] || return 0

  extract_urls "${head:-HEAD}" -- "${files[@]}"

}

extract_urls() {

  $(type -p ggrep grep | head -1) -hoe 'http[^"]\+#[^"]\+' "$@" | sort | uniq

  git grep -hoe 'http[^"]\+#[^"]\+' "$@" | sort | uniq

}

update() {

  local url

  local checksum

  local file

  local tmp_path

  for url in $(potentially_new_packages "$1"); do

    checksum="${url#*#}"

    url="${url%#*}"

    if test_mirrored "$checksum"; then

      echo "Already mirrored: $url"

    else

      echo "Mirroring: $url"

      file="${TMPDIR:-/tmp}/$checksum"

      echo "Will mirror: $url"

      [ -n "$tmp_path" ] || tmp_path="$(mktemp -d "${TMPDIR:-/tmp}/s3-sync.XXXXX")"

      file="$tmp_path/$checksum"

      download_and_verify "$url" "$file" "$checksum"

      ./script/s3-put "$file" "${AMAZON_S3_BUCKET?}"

    fi

  done

  if [ -n "$tmp_path" ]; then

    echo "Uploading..."

    aws s3 sync --acl=public-read --size-only "$tmp_path" "s3://${AMAZON_S3_BUCKET?}"

  fi

}

verify() {

  local url

  local checksum

  local file

  local status=0

  for url in $(potentially_new_packages "$1"); do

    checksum="${url#*#}"

    url="${url%#*}"

    echo "Verifying checksum for $url"

    file="${TMPDIR:-/tmp}/$checksum"

    download_and_verify "$url" "$file" "$checksum"

    download_and_verify "$url" "$file" "$checksum" || status=$?

  done

  return $status

}

stats() {

  local packages=( $(extract_urls ./share/ruby-build/*) )

  local packages

  IFS=$'\n' read -d '' -r -a packages < <(extract_urls -- ./share/ruby-build/\*)

  local total="${#packages[@]}"

  local confirmed=0

  local checksum

  for url in "${packages[@]}"; do

    checksum="${url#*#}"

    if test_mirrored "$checksum"; then

      confirmed="$((confirmed + 1))"

      : $((confirmed++))

    else

      echo "failed: $url" >&2

    fi

#!/usr/bin/env bash

# Usage: s3-put <FILE> <S3_BUCKET> [<CONTENT_TYPE>]

#

# Uploads a file to the Amazon S3 service.

#

# Depends on AWS credentials being set via env:

# - AMAZON_ACCESS_KEY_ID

# - AMAZON_SECRET_ACCESS_KEY

#

# Outputs the URL of the newly uploaded file.

set -e

authorization() {

  local signature="$(string_to_sign | hmac_sha1 | base64)"

  echo "AWS ${AMAZON_ACCESS_KEY_ID?}:${signature}"

}

hmac_sha1() {

  openssl dgst -binary -sha1 -hmac "${AMAZON_SECRET_ACCESS_KEY?}"

}

base64() {

  openssl enc -base64

}

bin_md5() {

  openssl dgst -binary -md5

}

string_to_sign() {

  echo "$http_method"

  echo "$content_md5"

  echo "$content_type"

  echo "$date"

  echo "x-amz-acl:$acl"

  printf "/$bucket/$remote_path"

}

date_string() {

  LC_TIME=C date "+%a, %d %h %Y %T %z"

}

file="$1"

bucket="$2"

content_type="$3"

http_method=PUT

acl="public-read"

remote_path="${file##*/}"

content_md5="$(bin_md5 < "$file" | base64)"

date="$(date_string)"

url="https://$bucket.s3.amazonaws.com/$remote_path"

curl -qsSf -T "$file" \

  -H "Authorization: $(authorization)" \

  -H "x-amz-acl: $acl" \

  -H "Date: $date" \

  -H "Content-MD5: $content_md5" \

  -H "Content-Type: $content_type" \

  "$url"

echo "$url"