Unverified Commit 93c50bba authored by Mislav Marohnić's avatar Mislav Marohnić
Browse files

Show progress of downloaded files in the terminal 

This connects the stderr of download utilities like curl and wget to the
original stderr of the process so that they can detect a terminal and
print progress bars to it.
parent 61e50df5

bin/ruby-build

+21 −33
Original line number Diff line number Diff line
@@ -406,6 +406,7 @@ http() { 
  RUBY_BUILD_HTTP_CLIENT="${RUBY_BUILD_HTTP_CLIENT:-$(detect_http_client 2>&3)}"

  [ -n "$RUBY_BUILD_HTTP_CLIENT" ] || return 1



  # http_get_curl, http_get_wget, etc.

  "http_${method}_${RUBY_BUILD_HTTP_CLIENT}" "$@"

}
@@ -423,37 +424,32 @@ detect_http_client() { 


http_head_aria2c() {

  # shellcheck disable=SC2086

  aria2c --dry-run --no-conf=true ${ARIA2_OPTS} "$1" >&4 2>&1

  aria2c --dry-run --no-conf=true $ARIA2_OPTS "$1" >/dev/null

}



http_get_aria2c() {

  local out="${2:-$(mktemp "out.XXXXXX")}"

  # shellcheck disable=SC2086

  if aria2c --allow-overwrite=true --no-conf=true -o "${out}" ${ARIA2_OPTS} "$1" >&4; then

    [ -n "$2" ] || cat "${out}"

  else

    false

  fi

  log_command aria2c --allow-overwrite=true --no-conf=true --console-log-level=warn --stderr $ARIA2_OPTS -o "$2" "$1" 2>&3

}



http_head_curl() {

  # shellcheck disable=SC2086

  curl -qsILf ${CURL_OPTS} "$1" >&4 2>&1

  curl -qsILf $CURL_OPTS "$1" >/dev/null

}



http_get_curl() {

  # shellcheck disable=SC2086

  curl -q -o "${2:--}" -sSLf ${CURL_OPTS} "$1"

  log_command curl -q -fL $CURL_OPTS -o "$2" "$1" 2>&3

}



http_head_wget() {

  # shellcheck disable=SC2086

  wget -q --spider ${WGET_OPTS} "$1" >&4 2>&1

  wget -q --spider $WGET_OPTS "$1" >/dev/null

}



http_get_wget() {

  # shellcheck disable=SC2086

  wget -nv ${WGET_OPTS} -O "${2:--}" "$1"

  log_command wget -nv --show-progress $WGET_OPTS -O "$2" "$1" 2>&3

}



fetch_tarball() {
@@ -494,26 +490,21 @@ fetch_tarball() { 
  if ! reuse_existing_tarball "$package_filename" "$checksum"; then

    local tarball_filename

    tarball_filename="$(basename "$package_url")"

    echo "Downloading ${tarball_filename}..." >&2

    log_info "Downloading ${tarball_filename}..."

    # shellcheck disable=SC2015

    http head "$mirror_url" &&

    download_tarball "$mirror_url" "$package_filename" "$checksum" ||

    download_tarball "$package_url" "$package_filename" "$checksum"

  fi



  { if tar "$tar_args" "$package_filename"; then

  log_command tar "$tar_args" "$package_filename" >/dev/null



  if [ ! -d "$package_name" ]; then

    extracted_dir="$(find_extracted_directory)"

    mv "$extracted_dir" "$package_name"

  fi



      if [ -z "$KEEP_BUILD_PATH" ]; then

        rm -f "$package_filename"

      else

        true

      fi

    fi

  } >&4 2>&1

  [ -n "$KEEP_BUILD_PATH" ] || rm -f "$package_filename"

}



find_extracted_directory() {
@@ -541,8 +532,8 @@ reuse_existing_tarball() { 
  local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename"



  [ -e "$cached_package_filename" ] || return 1

  verify_checksum "$cached_package_filename" "$checksum" >&4 2>&1 || return 1

  ln -s "$cached_package_filename" "$package_filename" >&4 2>&1 || return 1

  verify_checksum "$cached_package_filename" "$checksum" || return 1

  ln -s "$cached_package_filename" "$package_filename" || return 1

}



download_tarball() {
@@ -552,10 +543,8 @@ download_tarball() { 
  local package_filename="$2"

  local checksum="$3"



  echo "-> $package_url" >&2



  if http get "$package_url" "$package_filename" >&4 2>&1; then

    verify_checksum "$package_filename" "$checksum" >&4 2>&1 || return 1

  if http get "$package_url" "$package_filename"; then

    verify_checksum "$package_filename" "$checksum" || return 1

  else

    echo "error: failed to download $package_filename" >&2

    return 1
@@ -563,9 +552,8 @@ download_tarball() { 


  if [ -n "$RUBY_BUILD_CACHE_PATH" ]; then

    local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename"

    { mv "$package_filename" "$cached_package_filename"

    mv "$package_filename" "$cached_package_filename"

    ln -s "$cached_package_filename" "$package_filename"

    } >&4 2>&1 || return 1

  fi

}

test/cache.bats

+3 −3
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ export RUBY_BUILD_SKIP_MIRROR=1 
export RUBY_BUILD_CACHE_PATH="$TMP/cache"



@test "packages are saved to download cache" {

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  mkdir -p "$RUBY_BUILD_CACHE_PATH"

  install_fixture definitions/without-checksum
@@ -56,7 +56,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache" 


  stub shasum true "echo invalid" "echo $checksum"

  stub curl "-*I* : true" \

    "-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3"

    "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"



  mkdir -p "$RUBY_BUILD_CACHE_PATH"

  touch "${RUBY_BUILD_CACHE_PATH}/package-1.0.0.tar.gz"
@@ -74,7 +74,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache" 




@test "nonexistent cache directory is ignored" {

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  export RUBY_BUILD_CACHE_PATH="${TMP}/nonexistent"

test/checksum.bats

+9 −9
Original line number Diff line number Diff line
@@ -6,7 +6,7 @@ export RUBY_BUILD_CACHE_PATH= 




@test "package URL without checksum" {

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/without-checksum
@@ -19,7 +19,7 @@ export RUBY_BUILD_CACHE_PATH= 


@test "package URL with valid checksum" {

  stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum
@@ -33,7 +33,7 @@ export RUBY_BUILD_CACHE_PATH= 


@test "package URL with invalid checksum" {

  stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-invalid-checksum
@@ -47,7 +47,7 @@ export RUBY_BUILD_CACHE_PATH= 


@test "package URL with checksum but no shasum support" {

  stub shasum false

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum
@@ -61,7 +61,7 @@ export RUBY_BUILD_CACHE_PATH= 


@test "package URL with valid md5 checksum" {

  stub md5 true "echo 83e6d7725e20166024a1eb74cde80677"

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-md5-checksum
@@ -75,7 +75,7 @@ export RUBY_BUILD_CACHE_PATH= 


@test "package URL with md5 checksum but no md5 support" {

  stub md5 false

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-md5-checksum
@@ -89,7 +89,7 @@ export RUBY_BUILD_CACHE_PATH= 


@test "package with invalid checksum" {

  stub shasum true "echo invalid"

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum
@@ -125,7 +125,7 @@ DEF 
  stub shasum true \

    "echo invalid" \

    "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  export -n RUBY_BUILD_CACHE_PATH

  export RUBY_BUILD_BUILD_PATH="${TMP}/build"
@@ -144,7 +144,7 @@ DEF 
}



@test "package URL with checksum of unexpected length" {

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  run_inline_definition <<DEF

install_package "package-1.0.0" "http://example.com/packages/package-1.0.0.tar.gz#checksum_of_unexpected_length" copy

test/fetch.bats

+4 −18
Original line number Diff line number Diff line
@@ -14,7 +14,6 @@ setup() { 


  install_fixture definitions/without-checksum

  assert_failure

  assert_output_contains "> http://example.com/packages/package-1.0.0.tar.gz"

  assert_output_contains "error: failed to download package-1.0.0.tar.gz"

}
@@ -30,16 +29,11 @@ setup() { 
@test "using aria2c if available" {

  export RUBY_BUILD_ARIA2_OPTS=

  export -n RUBY_BUILD_HTTP_CLIENT

  stub aria2c "--allow-overwrite=true --no-conf=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"

  stub aria2c "--allow-overwrite=true --no-conf=true --console-log-level=warn --stderr -o * http://example.com/* : cp $FIXTURE_ROOT/\${7##*/} \$6"



  install_fixture definitions/without-checksum

  assert_success

  assert_output <<OUT

Downloading package-1.0.0.tar.gz...

-> http://example.com/packages/package-1.0.0.tar.gz

Installing package-1.0.0...

Installed package-1.0.0 to ${TMP}/install

OUT

  assert_output_contains "Downloading package-1.0.0.tar.gz..."

  unstub aria2c

}
@@ -50,11 +44,7 @@ OUT 
install_git "package-dev" "http://example.com/packages/package.git" master copy

DEF

  assert_success

  assert_output <<OUT

Cloning http://example.com/packages/package.git...

Installing package-dev...

Installed package-dev to ${TMP}/install

OUT

  assert_output_contains "Cloning http://example.com/packages/package.git..."

  unstub git

}
@@ -68,10 +58,6 @@ OUT 
install_git "package-dev" "http://example.com/packages/package.git" master copy

DEF

  assert_success

  assert_output <<OUT

Cloning http://example.com/packages/package.git...

Installing package-dev...

Installed package-dev to ${TMP}/install

OUT

  assert_output_contains "Cloning http://example.com/packages/package.git..."

  unstub git

}

test/mirror.bats

+12 −15
Original line number Diff line number Diff line
@@ -8,10 +8,9 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com 


@test "package URL without checksum bypasses mirror" {

  stub shasum true

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/without-checksum

  echo "$output" >&2



  assert_success

  assert [ -x "${INSTALL_ROOT}/bin/package" ]
@@ -23,7 +22,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com 


@test "package URL with checksum but no shasum support bypasses mirror" {

  stub shasum false

  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum
@@ -41,7 +40,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com 


  stub shasum true "echo $checksum"

  stub curl "-*I* $mirror_url : true" \

    "-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3"

    "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"



  install_fixture definitions/with-checksum
@@ -59,7 +58,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com 


  stub shasum true "echo $checksum"

  stub curl "-*I* $mirror_url : false" \

    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum
@@ -77,11 +76,10 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com 


  stub shasum true "echo invalid" "echo $checksum"

  stub curl "-*I* $mirror_url : true" \

    "-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \

    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

    "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \

    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum

  echo "$output" >&2



  assert_success

  assert [ -x "${INSTALL_ROOT}/bin/package" ]
@@ -97,7 +95,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com 


  stub shasum true "echo $checksum"

  stub curl "-*I* : true" \

    "-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \

    "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \



  install_fixture definitions/with-checksum
@@ -114,7 +112,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com 
  local checksum="ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"



  stub shasum true "echo $checksum"

  stub curl "-q -o * -*S* https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

  stub curl "-q -fL -o * https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  run_inline_definition <<DEF

install_package "package-1.0.0" "https://cache.ruby-lang.org/packages/package-1.0.0.tar.gz#ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" copy
@@ -135,7 +133,7 @@ DEF 


  stub shasum true "echo $checksum"

  stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \

    "-q -o * -*S* $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3"

    "-q -fL -o * $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"



  install_fixture definitions/with-checksum
@@ -154,7 +152,7 @@ DEF 


  stub shasum true "echo $checksum"

  stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : false" \

    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum
@@ -173,11 +171,10 @@ DEF 


  stub shasum true "echo invalid" "echo $checksum"

  stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \

    "-q -o * -*S* $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \

    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"

    "-q -fL -o * $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \

    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"



  install_fixture definitions/with-checksum

  echo "$output" >&2



  assert_success

  assert [ -x "${INSTALL_ROOT}/bin/package" ]