From 1d888b956f989f6646e6ec79bdf6612b8b17bed3 Mon Sep 17 00:00:00 2001 From: Ed J Date: Sun, 29 Oct 2017 01:57:03 +0000 Subject: [PATCH 1/3] implement local::lib caching --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++ bin/local-lib | 13 +++++++---- bin/local-lib-cache | 41 +++++++++++++++++++++++++++++++++ init | 5 +++++ 4 files changed, 110 insertions(+), 4 deletions(-) create mode 100755 bin/local-lib-cache diff --git a/README.md b/README.md index 36e407c..bdef7cf 100644 --- a/README.md +++ b/README.md @@ -116,8 +116,17 @@ There are various environment variables that will control how a build is done. generation process if it has heavy dependencies, such as Dist::Zilla, and testing is being done on a freshly built perl. + Makes a local::lib that is blank. + Defaults to true. + * `LOCAL_LIB_CACHE` + + The directory used to cache the local::lib used if you specify + `local-lib cache`. + + Defaults to `$HOME/.perlbrew-cache`. + * `CPAN_MIRROR` The CPAN mirror that dependencies will be installed from. @@ -278,6 +287,52 @@ Commands or built if not, and added to the library path. This is taken care of by the `build-perl` step. + There is a special local::lib name you can give: + `cache`, which works together with [Travis's caching + feature](https://docs.travis-ci.com/user/caching). Its location + defaults to `$HOME/.perlbrew-cache` but can be overridden + by supplying the environment variable `LOCAL_LIB_CACHE`. Use it + like this: + + cache: + directories: + - $HOME/.perlbrew-cache # keeps between builds here + perl: + - "5.8.4@moo" + before_install: + - git clone git://github.com/travis-perl/helpers ~/travis-perl-helpers + - source ~/travis-perl-helpers/init + - build-perl + - local-lib cache # makes a local::lib from cached + - perl -V + - build-dist + - cd $BUILD_DIR + install: + - cpan-install --deps # installs prereqs, including recommends + before_cache: + - local-lib-cachestore # saves for local::lib next time + + Alternatively, this will also work and the `build-perl` will do the + `local-lib` step for you: + + cache: + directories: + - $HOME/.perlbrew-cache # keeps between builds here + perl: + - "5.8.4@cache" + before_install: + - git clone git://github.com/travis-perl/helpers ~/travis-perl-helpers + - source ~/travis-perl-helpers/init + - build-perl + # no need for local-lib cache + - perl -V + - build-dist + - cd $BUILD_DIR + install: + - cpan-install --deps # installs prereqs, including recommends + before_cache: + - local-lib-cachestore # saves for local::lib next time + local::lib ---------- A number of [predefined local::lib](share/local-libs.txt) directories are diff --git a/bin/local-lib b/bin/local-lib index 86e4157..292e787 100755 --- a/bin/local-lib +++ b/bin/local-lib @@ -7,12 +7,17 @@ if [ -z "$1" ]; then ll_roots=("$(mktemp -d -t local-lib-XXXXXX)") else for lib; do - if [ -z "$LL_NO_INSTALL" ]; then - # redirect since output of this gets eval-ed - if $HELPERS_ROOT/bin/local-lib-installer "$lib" 1>&2; then - ll_roots+=("$PERLBREW_HOME/libs/$PERLBREW_PERL@$lib") + if [ "$lib" == "cache" ]; then + $HELPERS_ROOT/bin/local-lib-cache use + else + if [ -z "$LL_NO_INSTALL" ]; then + # redirect since output of this gets eval-ed + if $HELPERS_ROOT/bin/local-lib-installer "$lib" 1>&2; then + ll_roots+=("$PERLBREW_HOME/libs/$PERLBREW_PERL@$lib") + fi fi fi + ll_roots+=("$PERLBREW_HOME/libs/$PERLBREW_PERL@$lib") done fi diff --git a/bin/local-lib-cache b/bin/local-lib-cache new file mode 100755 index 0000000..04506ff --- /dev/null +++ b/bin/local-lib-cache @@ -0,0 +1,41 @@ +#!/bin/bash + +# usage: +# $0 use +# copies from cache location to perlbrew location +# $0 store +# copies from perlbrew location to cache location + +# locations: +cache_location="$LOCAL_LIB_CACHE" +full_version="$PERLBREW_PERL@cache" +perlbrew_location="$PERLBREW_HOME/libs/$full_version" + +[ -z "$HELPERS_ROOT" ] && export HELPERS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +source "$HELPERS_ROOT/lib/debug.sh" +exec 1>&2 # since local-lib calls this, then evals stdout - any echo to stderr + +if [ -z "$PERLBREW_ROOT" ]; then + echo "Must be run under perlbrew!" + exit 1 +fi + +source "$PERLBREW_ROOT/etc/bashrc" + +set -e + +if [ "$1" == "use" ]; then + perlbrew lib create "$full_version" + perlbrew use "$full_version" + mkdir -p "$perlbrew_location" + cp -pr "$cache_location"/* "$perlbrew_location" +elif [ "$1" == "store" ]; then + id + echo "BEFORE"; find "$perlbrew_location" "$cache_location" -ls + mkdir -p "$cache_location" + cp -pr "$perlbrew_location"/* "$cache_location" + echo "AFTER"; find "$cache_location" -ls +else + echo "Unknown arg '$1'" + exit 1 +fi diff --git a/init b/init index d35fd1e..9150c28 100644 --- a/init +++ b/init @@ -19,6 +19,7 @@ export SYSTEM_CORES="$(grep -c ^processor /proc/cpuinfo 2>/dev/null || sysctl -n [ -z "$AUTHOR_TESTING" ] && export AUTHOR_TESTING=1 [ -z "$SPLIT_BUILD" ] && export SPLIT_BUILD=1 [ "$COVERAGE" == "1" ] && export COVERAGE=coveralls # drop this eventually +[ -z "$LOCAL_LIB_CACHE" ] && export LOCAL_LIB_CACHE="$HOME/.perlbrew-cache" [ -z "$PERL_AUTOINSTALL_PREFER_CPAN" ] && export PERL_AUTOINSTALL_PREFER_CPAN=1 [ -z "$PERL_MM_USE_DEFAULT" ] && export PERL_MM_USE_DEFAULT=1 [ -z "$NONINTERACTIVE_TESTING" ] && export NONINTERACTIVE_TESTING=1 @@ -52,6 +53,10 @@ function local-lib { eval "$("$HELPERS_ROOT/bin/local-lib" "$@")" } +function local-lib-cachestore { + $HELPERS_ROOT/bin/local-lib-cache store +} + function build-perl { local TRAVIS_PERL_VERSION="$TRAVIS_PERL_VERSION" if [[ ( "$TRAVIS_PERL_VERSION" =~ ^5\.6[.-] || \ From ecd145ee1a694cdceed04d6b913e9ab1cf2c3fc7 Mon Sep 17 00:00:00 2001 From: Ed J Date: Sun, 29 Oct 2017 05:26:05 +0000 Subject: [PATCH 2/3] use symlink not copying for cache-use --- README.md | 4 +--- bin/local-lib-cache | 12 ++++-------- init | 4 ---- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index bdef7cf..e4fc27e 100644 --- a/README.md +++ b/README.md @@ -303,14 +303,12 @@ Commands - git clone git://github.com/travis-perl/helpers ~/travis-perl-helpers - source ~/travis-perl-helpers/init - build-perl - - local-lib cache # makes a local::lib from cached + - local-lib cache # makes a local::lib symlinked to cache - perl -V - build-dist - cd $BUILD_DIR install: - cpan-install --deps # installs prereqs, including recommends - before_cache: - - local-lib-cachestore # saves for local::lib next time Alternatively, this will also work and the `build-perl` will do the `local-lib` step for you: diff --git a/bin/local-lib-cache b/bin/local-lib-cache index 04506ff..77cd5b0 100755 --- a/bin/local-lib-cache +++ b/bin/local-lib-cache @@ -27,14 +27,10 @@ set -e if [ "$1" == "use" ]; then perlbrew lib create "$full_version" perlbrew use "$full_version" - mkdir -p "$perlbrew_location" - cp -pr "$cache_location"/* "$perlbrew_location" -elif [ "$1" == "store" ]; then - id - echo "BEFORE"; find "$perlbrew_location" "$cache_location" -ls - mkdir -p "$cache_location" - cp -pr "$perlbrew_location"/* "$cache_location" - echo "AFTER"; find "$cache_location" -ls + rm -rf "$perlbrew_location" + ln -s "$cache_location" "$perlbrew_location" + [ -n "$TRAVIS_PERL_DEBUG" ] && \ + (id; echo "ON USE"; find "$perlbrew_location" "$cache_location" -ls) else echo "Unknown arg '$1'" exit 1 diff --git a/init b/init index 9150c28..4673f79 100644 --- a/init +++ b/init @@ -53,10 +53,6 @@ function local-lib { eval "$("$HELPERS_ROOT/bin/local-lib" "$@")" } -function local-lib-cachestore { - $HELPERS_ROOT/bin/local-lib-cache store -} - function build-perl { local TRAVIS_PERL_VERSION="$TRAVIS_PERL_VERSION" if [[ ( "$TRAVIS_PERL_VERSION" =~ ^5\.6[.-] || \ From cfeb321edafeb6fe1a9934f2ec3b0af9e43ecc37 Mon Sep 17 00:00:00 2001 From: Ed J Date: Sun, 29 Oct 2017 06:49:25 +0000 Subject: [PATCH 3/3] handle SPLITBUILD with separately cacheable dir --- README.md | 4 +++- bin/local-lib | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4fc27e..b95b41e 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,8 @@ There are various environment variables that will control how a build is done. generation process if it has heavy dependencies, such as Dist::Zilla, and testing is being done on a freshly built perl. - Makes a local::lib that is blank. + Makes a local::lib that symlinks to `${LOCAL_LIB_CACHE}-split`. If you + give that as a cached directory, it will make your builds go faster. Defaults to true. @@ -297,6 +298,7 @@ Commands cache: directories: - $HOME/.perlbrew-cache # keeps between builds here + - $HOME/.perlbrew-cache-split # used by split builds perl: - "5.8.4@moo" before_install: diff --git a/bin/local-lib b/bin/local-lib index 292e787..40b5604 100755 --- a/bin/local-lib +++ b/bin/local-lib @@ -4,7 +4,9 @@ source "$HELPERS_ROOT/lib/debug.sh" declare -a ll_roots if [ -z "$1" ]; then - ll_roots=("$(mktemp -d -t local-lib-XXXXXX)") + ll_roots=("${LOCAL_LIB_CACHE}-split") # can cache + [ -n "$TRAVIS_PERL_DEBUG" ] && \ + (echo "ON SPLITDIR"; find "${LOCAL_LIB_CACHE}-split" -ls) 1>&2 else for lib; do if [ "$lib" == "cache" ]; then