Merge pull request #438 from sstephenson/travis-mirroring

install: git clone https://github.com/sstephenson/bats.git

script: bats/bin/bats -t test

# skips unnecessary Ruby-specific setup

script: script/test

language: c

env:

  global:

    - AMAZON_S3_BUCKET=ruby-build-mirror

    - AMAZON_ACCESS_KEY_ID=AKIAJKAUQVHU6X4CODDQ

    - secure: LTSvDP2o72nbECDwWsfwnsiETF4VpqrYN3y/ve68AZIMzfNWDB5vhqzMLU1ltFnSNxd71gTCGX2OEcsxdrfnG+Msu52v8FtJ7lz/b9xn83gGYrGnmEMzARtd1fnuzlWQh/1eNL9jrNl8FDhgjoTqKl2gF6fZBsQxcHRnvRSXcqE=

#!/usr/bin/env bash

# Usage: script/mirror update <COMMIT-RANGE>

#        script/mirror stats

set -e

commit_range="${1?}"

eval "$(grep RUBY_BUILD_MIRROR_URL= ./bin/ruby-build | head -1)"

test_mirrored() {

  curl -qsSfIL "$RUBY_BUILD_MIRROR_URL/$1" >/dev/null 2>&1

}

compute_md5() {

  local output="$(openssl md5)"

  echo "${output##* }" | tr '[A-Z]' '[a-z]'

}

download_package() {

  curl -qsSfL -o "$2" "$1"

}

download_and_verify() {

  local checksum

  local url="$1"

  local file="$2"

  local expected="$3"

  download_package "$url" "$file"

  checksum="$(compute_md5 < "$file")"

  if [ "$checksum" != "$expected" ]; then

    echo "Error: $url doesn't match its checksum $expected" >&2

    return 1

  fi

}

changed_files() {

  git diff --name-only --diff-filter=ACMR "$@"

}

potentially_new_packages() {

  local files="$(changed_files "$1" -- ./share/ruby-build)"

  [ -n "$files" ] && extract_urls $files

}

extract_urls() {

  grep -hoe 'http[^"]\+#[^"]\+' "$@"

}

update() {

  local url

  local checksum

  local file

  for url in $(potentially_new_packages "$1"); do

    checksum="${url#*#}"

    url="${url%#*}"

    if test_mirrored "$checksum"; then

      echo "Already mirrored: $url"

    else

      echo "Mirroring: $url"

      file="${TMPDIR:-/tmp}/$checksum"

      download_and_verify "$url" "$file" "$checksum"

      ./script/s3-put "$file" "${AMAZON_S3_BUCKET?}"

    fi

  done

}

stats() {

  local packages=( $(extract_urls ./share/ruby-build/*) )

  local total="${#packages[@]}"

  local confirmed=0

  local checksum

  for url in "${packages[@]}"; do

    checksum="${url#*#}"

    if test_mirrored "$checksum"; then

      confirmed="$((confirmed + 1))"

    else

      echo "failed: $url" >&2

    fi

    echo -n "."

  done

  echo

  echo "$confirmed/$total mirrored"

}

cmd="${1?}"

shift 1

"$cmd" "$@"

#!/usr/bin/env bash

# Usage: s3-put <FILE> <S3_BUCKET> [<CONTENT_TYPE>]

#

# Uploads a file to the Amazon S3 service.

#

# Depends on AWS credentials being set via env:

# - AMAZON_ACCESS_KEY_ID

# - AMAZON_SECRET_ACCESS_KEY

#

# Outputs the URL of the newly uploaded file.

set -e

authorization() {

  local signature="$(string_to_sign | hmac_sha1 | base64)"

  echo "AWS ${AMAZON_ACCESS_KEY_ID?}:${signature}"

}

hmac_sha1() {

  openssl dgst -binary -sha1 -hmac "${AMAZON_SECRET_ACCESS_KEY?}"

}

base64() {

  openssl enc -base64

}

bin_md5() {

  openssl dgst -binary -md5

}

string_to_sign() {

  echo "$http_method"

  echo "$content_md5"

  echo "$content_type"

  echo "$date"

  echo "x-amz-acl:$acl"

  printf "/$bucket/$remote_path"

}

date_string() {

  LC_TIME=C date "+%a, %d %h %Y %T %z"

}

file="$1"

bucket="$2"

content_type="$3"

http_method=PUT

acl="public-read"

remote_path="${file##*/}"

content_md5="$(bin_md5 < "$file" | base64)"

date="$(date_string)"

url="https://$bucket.s3.amazonaws.com/$remote_path"

curl -qsSf -T "$file" \

  -H "Authorization: $(authorization)" \

  -H "x-amz-acl: $acl" \

  -H "Date: $date" \

  -H "Content-MD5: $content_md5" \

  -H "Content-Type: $content_type" \

  "$url"

echo "$url"

#!/usr/bin/env bash

set -e

[ -d ./bats/bin ] && export PATH=./bats/bin:"$PATH"

bats -t test

if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then

  ./script/mirror update "$TRAVIS_COMMIT_RANGE"

fi