Clean up the base dockerfile, build C deps concurrently.

This commit is contained in:
Péter Szilágyi 2015-05-08 12:21:51 +03:00
parent 84f5af0336
commit a1bc698073
5 changed files with 218 additions and 146 deletions

View File

@ -7,18 +7,15 @@ FROM ubuntu:14.04
MAINTAINER Péter Szilágyi <peterke@gmail.com>
# Create a small script to download binaries and validate their checksum
ENV FETCH ./fetch.sh
RUN \
echo '#!/bin/bash' > $FETCH && \
echo 'set -e' >> $FETCH && \
echo 'file=`basename $1`' >> $FETCH && \
echo 'echo "Downloading $1..."' >> $FETCH && \
echo 'wget -q $1' >> $FETCH && \
echo 'echo "$2 $file" > $file.sum' >> $FETCH && \
echo 'sha1sum -c $file.sum' >> $FETCH && \
echo 'rm $file.sum' >> $FETCH && \
chmod +x $FETCH
# Configure the Go environment, since it's not going to change
ENV PATH /usr/local/go/bin:$PATH
ENV GOPATH /go
# Inject the remote file fetcher and checksum verifier
ADD fetch.sh /fetch.sh
ENV FETCH /fetch.sh
RUN chmod +x $FETCH
# Make sure apt-get is up to date and dependent packages are installed
@ -46,141 +43,19 @@ RUN \
ENV PATH /osxcross/target/bin:$PATH
# Create a small script to download all the Go packages, install the core Linux
# package, inject and bootstrap the others
ENV BOOTSTRAP ./bootstrap.sh
RUN \
echo '#!/bin/bash' > $BOOTSTRAP && \
echo 'set -e' >> $BOOTSTRAP && \
echo >> $BOOTSTRAP && \
echo '$FETCH $DIST_LINUX_64 $DIST_LINUX_64_SHA1' >> $BOOTSTRAP && \
echo '$FETCH $DIST_LINUX_32 $DIST_LINUX_32_SHA1' >> $BOOTSTRAP && \
echo '$FETCH $DIST_LINUX_ARM $DIST_LINUX_ARM_SHA1' >> $BOOTSTRAP && \
echo '$FETCH $DIST_OSX_64 $DIST_OSX_64_SHA1' >> $BOOTSTRAP && \
echo '$FETCH $DIST_OSX_32 $DIST_OSX_32_SHA1' >> $BOOTSTRAP && \
echo '$FETCH $DIST_WIN_64 $DIST_WIN_64_SHA1' >> $BOOTSTRAP && \
echo '$FETCH $DIST_WIN_32 $DIST_WIN_32_SHA1' >> $BOOTSTRAP && \
echo >> $BOOTSTRAP && \
echo 'tar -C /usr/local -xzf `basename $DIST_LINUX_64`' >> $BOOTSTRAP && \
echo >> $BOOTSTRAP && \
echo 'tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_32` go/pkg/linux_386*' >> $BOOTSTRAP && \
echo 'GOOS=linux GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap' >> $BOOTSTRAP && \
echo 'tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_ARM` go/pkg/linux_arm*' >> $BOOTSTRAP && \
echo 'GOOS=linux GOARCH=arm /usr/local/go/pkg/tool/linux_amd64/dist bootstrap' >> $BOOTSTRAP && \
echo >> $BOOTSTRAP && \
echo 'tar -C /usr/local --wildcards -xzf `basename $DIST_OSX_64` go/pkg/darwin_amd64*' >> $BOOTSTRAP && \
echo 'GOOS=darwin GOARCH=amd64 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap' >> $BOOTSTRAP && \
echo 'tar -C /usr/local --wildcards -xzf `basename $DIST_OSX_32` go/pkg/darwin_386*' >> $BOOTSTRAP && \
echo 'GOOS=darwin GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap' >> $BOOTSTRAP && \
echo >> $BOOTSTRAP && \
echo 'unzip -d /usr/local -q `basename $DIST_WIN_64` go/pkg/windows_amd64*' >> $BOOTSTRAP && \
echo 'GOOS=windows GOARCH=amd64 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap' >> $BOOTSTRAP && \
echo 'unzip -d /usr/local -q `basename $DIST_WIN_32` go/pkg/windows_386*' >> $BOOTSTRAP && \
echo 'GOOS=windows GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap' >> $BOOTSTRAP && \
echo >> $BOOTSTRAP && \
echo 'rm -f `basename $DIST_LINUX_64` `basename $DIST_LINUX_32` `basename $DIST_LINUX_ARM` \\' >> $BOOTSTRAP && \
echo ' `basename $DIST_OSX_64` `basename $DIST_OSX_32` `basename $DIST_WIN_64` `basename $DIST_WIN_32`' >> $BOOTSTRAP && \
chmod +x $BOOTSTRAP
# Inject the Go package downloader and tool-chain bootstrapper
ADD bootstrap.sh /bootstrap.sh
ENV BOOTSTRAP /bootstrap.sh
RUN chmod +x $BOOTSTRAP
ENV PATH /usr/local/go/bin:$PATH
ENV GOPATH /go
# Create a small script to iterate over the dependencies and cross compile them
# Inject the C dependency cross compiler
ADD build_deps.sh /build_deps.sh
ENV BUILD_DEPS /build_deps.sh
RUN \
echo '#!/bin/bash' > $BUILD_DEPS && \
echo 'set -e' >> $BUILD_DEPS && \
echo >> $BUILD_DEPS && \
echo 'rm -rf /deps-build && cp -r /deps /deps-build' >> $BUILD_DEPS && \
echo 'for dep in `ls /deps-build`; do' >> $BUILD_DEPS && \
echo ' cd /deps-build/$dep && ./configure --disable-shared --host=$HOST --prefix=$PREFIX && make install' >> $BUILD_DEPS && \
echo 'done' >> $BUILD_DEPS && \
chmod +x $BUILD_DEPS
RUN chmod +x $BUILD_DEPS
# Inject the container entry point, the build script
ADD build.sh /build.sh
ENV BUILD /build.sh
RUN chmod +x $BUILD
# Create a small script to go get a package and cross compile it
ENV BUILD ./build.sh
RUN \
echo '#!/bin/bash' > $BUILD && \
echo >> $BUILD && \
echo 'echo Fetching dependencies...' >> $BUILD && \
echo 'mkdir /deps' >> $BUILD && \
echo 'DEPS=($DEPS) && for dep in "${DEPS[@]}"; do' >> $BUILD && \
echo ' echo Downloading $dep' >> $BUILD && \
echo ' if [ "${dep##*.}" == "tar" ]; then wget $dep -O - | tar -C /deps -x; fi' >> $BUILD && \
echo ' if [ "${dep##*.}" == "gz" ]; then wget $dep -O - | tar -C /deps -xz; fi' >> $BUILD && \
echo ' if [ "${dep##*.}" == "bz2" ]; then wget $dep -O - | tar -C /deps -xj; fi' >> $BUILD && \
echo 'done' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Fetching $1...' >> $BUILD && \
echo 'go get -d $1' >> $BUILD && \
echo 'cd $GOPATH/src/$1' >> $BUILD && \
echo 'export GOPATH=$GOPATH:`pwd`/Godeps/_workspace' >> $BUILD && \
echo >> $BUILD && \
echo 'if [ "$REPO_REMOTE" != "" ]; then' >> $BUILD && \
echo ' if [ -d ".git" ]; then' >> $BUILD && \
echo ' git remote set-url origin $REPO_REMOTE' >> $BUILD && \
echo ' git pull' >> $BUILD && \
echo ' elif [ -d ".hg" ]; then' >> $BUILD && \
echo ' echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc' >> $BUILD && \
echo ' hg pull' >> $BUILD && \
echo ' fi' >> $BUILD && \
echo 'fi' >> $BUILD && \
echo >> $BUILD && \
echo 'if [ "$REPO_BRANCH" != "" ]; then' >> $BUILD && \
echo ' if [ -d ".git" ]; then' >> $BUILD && \
echo ' git checkout $REPO_BRANCH' >> $BUILD && \
echo ' elif [ -d ".hg" ]; then' >> $BUILD && \
echo ' hg checkout $REPO_BRANCH' >> $BUILD && \
echo ' fi' >> $BUILD && \
echo 'fi' >> $BUILD && \
echo >> $BUILD && \
echo 'NAME=`basename $1/$PACK`' >> $BUILD && \
echo 'if [ "$OUT" != "" ]; then' >> $BUILD && \
echo ' NAME=$OUT' >> $BUILD && \
echo 'fi' >> $BUILD && \
echo >> $BUILD && \
echo 'if [ "$FLAG_V" == "true" ]; then V=-v; fi' >> $BUILD && \
echo 'if [ "$FLAG_RACE" == "true" ]; then R=-race; fi' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Compiling for linux/amd64...' >> $BUILD && \
echo 'HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS' >> $BUILD && \
echo 'GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK' >> $BUILD && \
echo 'GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-linux-amd64$R ./$PACK' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Compiling for linux/386...' >> $BUILD && \
echo 'HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS' >> $BUILD && \
echo 'GOOS=linux GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK' >> $BUILD && \
echo 'GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-linux-386 ./$PACK' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Compiling for linux/arm...' >> $BUILD && \
echo 'CC=arm-linux-gnueabi-gcc HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS' >> $BUILD && \
echo 'CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go get -d ./$PACK' >> $BUILD && \
echo 'CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go build $V -o $NAME-linux-arm ./$PACK' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Compiling for windows/amd64...' >> $BUILD && \
echo 'CC=x86_64-w64-mingw32-gcc HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS' >> $BUILD && \
echo 'CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK' >> $BUILD && \
echo 'CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-windows-amd64$R.exe ./$PACK' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Compiling for windows/386...' >> $BUILD && \
echo 'CC=i686-w64-mingw32-gcc HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS' >> $BUILD && \
echo 'CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK' >> $BUILD && \
echo 'CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-windows-386.exe ./$PACK' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Compiling for darwin/amd64...' >> $BUILD && \
echo 'CC=o64-clang HOST=x86_64-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS' >> $BUILD && \
echo 'CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK' >> $BUILD && \
echo 'CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-darwin-amd64$R ./$PACK' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Compiling for darwin/386...' >> $BUILD && \
echo 'CC=o32-clang HOST=i386-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS' >> $BUILD && \
echo 'CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK' >> $BUILD && \
echo 'CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-darwin-386 ./$PACK' >> $BUILD && \
echo >> $BUILD && \
echo 'echo Moving binaries to host...' >> $BUILD && \
echo 'cp `ls -t | head -n 7` /build' >> $BUILD && \
chmod +x $BUILD
ENTRYPOINT ["./build.sh"]
ENTRYPOINT ["/build.sh"]

50
docker/base/bootstrap.sh Normal file
View File

@ -0,0 +1,50 @@
#!/bin/bash
#
# Contains the Go tool-chain bootstrapper, that retrieves all the configured
# distribution packages, extracts the binaries and deletes anything not needed.
#
# Usage: bootstrap.sh
#
# Needed environment variables:
# FETCH - Remote file fetcher and checksum verifier (injected by image)
# DIST_LINUX_64, DIST_LINUX_64_SHA1 - 64 bit Linux Go binaries and checksum
# DIST_LINUX_32, DIST_LINUX_32_SHA1 - 32 bit Linux Go binaries and checksum
# DIST_LINUX_ARM, DIST_LINUX_ARM_SHA1 - ARM v5 Linux Go binaries and checksum
# DIST_OSX_64, DIST_OSX_64_SHA1 - 64 bit Mac OSX Go binaries and checksum
# DIST_OSX_32, DIST_OSX_32_SHA1 - 32 bit Mac OSX Go binaries and checksum
# DIST_WIN_64, DIST_WIN_64_SHA1 - 64 bit Windows Go binaries and checksum
# DIST_WIN_32, DIST_WIN_32_SHA1 - 32 bit Windows Go binaries and checksum
set -e
# Download and verify all the binary packages
$FETCH $DIST_LINUX_64 $DIST_LINUX_64_SHA1
$FETCH $DIST_LINUX_32 $DIST_LINUX_32_SHA1
$FETCH $DIST_LINUX_ARM $DIST_LINUX_ARM_SHA1
$FETCH $DIST_OSX_64 $DIST_OSX_64_SHA1
$FETCH $DIST_OSX_32 $DIST_OSX_32_SHA1
$FETCH $DIST_WIN_64 $DIST_WIN_64_SHA1
$FETCH $DIST_WIN_32 $DIST_WIN_32_SHA1
# Extract the 64 bit Linux package as the primary Go SDK
tar -C /usr/local -xzf `basename $DIST_LINUX_64`
# Extract all other packages as secondary ones, keeping only the binaries
tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_32` go/pkg/linux_386*
GOOS=linux GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
tar -C /usr/local --wildcards -xzf `basename $DIST_LINUX_ARM` go/pkg/linux_arm*
GOOS=linux GOARCH=arm /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
tar -C /usr/local --wildcards -xzf `basename $DIST_OSX_64` go/pkg/darwin_amd64*
GOOS=darwin GOARCH=amd64 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
tar -C /usr/local --wildcards -xzf `basename $DIST_OSX_32` go/pkg/darwin_386*
GOOS=darwin GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
unzip -d /usr/local -q `basename $DIST_WIN_64` go/pkg/windows_amd64*
GOOS=windows GOARCH=amd64 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
unzip -d /usr/local -q `basename $DIST_WIN_32` go/pkg/windows_386*
GOOS=windows GOARCH=386 /usr/local/go/pkg/tool/linux_amd64/dist bootstrap
# Delete all the intermediate downloaded files
rm -f `basename $DIST_LINUX_64` `basename $DIST_LINUX_32` `basename $DIST_LINUX_ARM` \
`basename $DIST_OSX_64` `basename $DIST_OSX_32` \
`basename $DIST_WIN_64` `basename $DIST_WIN_32`

103
docker/base/build.sh Normal file
View File

@ -0,0 +1,103 @@
#!/bin/bash
#
# Contains the main cross compiler, that individually sets up each target build
# platform, compiles all the C dependencies, then build the requested executable
# itself.
#
# Usage: build.sh <import path>
#
# Needed environment variables:
# REPO_REMOTE - Optional VCS remote if not the primary repository is needed
# REPO_BRANCH - Optional VCS branch to use, if not the master branch
# DEPS - Optional list of C dependency packages to build
# PACK - Optional sub-package, if not the import path is being built
# OUT - Optional output prefix to override the package name
# FLAG_V - Optional verbosity flag to set on the Go builder
# FLAG_RACE - Optional race flag to set on the Go builder
# Download the canonical import path (may fail, don't allow failures beyond)
echo "Fetching main repository $1..."
go get -d $1
set -e
cd $GOPATH/src/$1
export GOPATH=$GOPATH:`pwd`/Godeps/_workspace
# Switch over the code-base to another checkout if requested
if [ "$REPO_REMOTE" != "" ]; then
echo "Switching over to remote $REPO_REMOTE..."
if [ -d ".git" ]; then
git remote set-url origin $REPO_REMOTE
git pull
elif [ -d ".hg" ]; then
echo -e "[paths]\ndefault = $REPO_REMOTE\n" >> .hg/hgrc
hg pull
fi
fi
if [ "$REPO_BRANCH" != "" ]; then
echo "Switching over to branch $REPO_BRANCH..."
if [ -d ".git" ]; then
git checkout $REPO_BRANCH
elif [ -d ".hg" ]; then
hg checkout $REPO_BRANCH
fi
fi
# Download all the C dependencies
echo "Fetching dependencies..."
mkdir /deps
DEPS=($DEPS) && for dep in "${DEPS[@]}"; do
echo Downloading $dep
if [ "${dep##*.}" == "tar" ]; then wget -q $dep -O - | tar -C /deps -x; fi
if [ "${dep##*.}" == "gz" ]; then wget -q $dep -O - | tar -C /deps -xz; fi
if [ "${dep##*.}" == "bz2" ]; then wget -q $dep -O - | tar -C /deps -xj; fi
done
# Configure some global build parameters
NAME=`basename $1/$PACK`
if [ "$OUT" != "" ]; then
NAME=$OUT
fi
if [ "$FLAG_V" == "true" ]; then V=-v; fi
if [ "$FLAG_RACE" == "true" ]; then R=-race; fi
# Build for each platform individually
echo "Compiling for linux/amd64..."
HOST=x86_64-linux PREFIX=/usr/local $BUILD_DEPS /deps
GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
GOOS=linux GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-linux-amd64$R ./$PACK
echo "Compiling for linux/386..."
HOST=i686-linux PREFIX=/usr/local $BUILD_DEPS /deps
GOOS=linux GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
GOOS=linux GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-linux-386 ./$PACK
echo "Compiling for linux/arm..."
CC=arm-linux-gnueabi-gcc HOST=arm-linux PREFIX=/usr/local/arm $BUILD_DEPS /deps
CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go get -d ./$PACK
CC=arm-linux-gnueabi-gcc GOOS=linux GOARCH=arm CGO_ENABLED=1 GOARM=5 go build $V -o $NAME-linux-arm ./$PACK
echo "Compiling for windows/amd64..."
CC=x86_64-w64-mingw32-gcc HOST=x86_64-w64-mingw32 PREFIX=/usr/x86_64-w64-mingw32 $BUILD_DEPS /deps
CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-windows-amd64$R.exe ./$PACK
echo "Compiling for windows/386..."
CC=i686-w64-mingw32-gcc HOST=i686-w64-mingw32 PREFIX=/usr/i686-w64-mingw32 $BUILD_DEPS /deps
CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
CC=i686-w64-mingw32-gcc GOOS=windows GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-windows-386.exe ./$PACK
echo "Compiling for darwin/amd64..."
CC=o64-clang HOST=x86_64-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go get -d ./$PACK
CC=o64-clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build $V $R -o $NAME-darwin-amd64$R ./$PACK
echo "Compiling for darwin/386..."
CC=o32-clang HOST=i386-apple-darwin10 PREFIX=/usr/local $BUILD_DEPS /deps
CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go get -d ./$PACK
CC=o32-clang GOOS=darwin GOARCH=386 CGO_ENABLED=1 go build $V -o $NAME-darwin-386 ./$PACK
echo "Moving binaries to host..."
cp `ls -t | head -n 7` /build

27
docker/base/build_deps.sh Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash
#
# Contains the a dependency builder to iterate over all installed dependencies
# and cross compile them to the requested target platform.
#
# Usage: build_deps.sh <dependency folder>
#
# Needed environment variables:
# CC - C cross compiler to use for the build
# HOST - Target platform to build (used to find the needed tool-chains)
# PREFIX - File-system path where to install the built binaries
set -e
# Remove any previous build leftovers, and copy a fresh working set (clean doesn't work for cross compiling)
rm -rf /deps-build && cp -r $1 /deps-build
# Build all the dependencies (no order for now)
for dep in `ls /deps-build`; do
echo "Configuring dependency $dep for $HOST..."
(cd /deps-build/$dep && ./configure --disable-shared --host=$HOST --prefix=$PREFIX --silent)
echo "Building dependency $dep for $HOST..."
(cd /deps-build/$dep && make --silent -j install)
done
# Remove any build artifacts
rm -rf /deps-build

17
docker/base/fetch.sh Normal file
View File

@ -0,0 +1,17 @@
#!/bin/bash
#
# Contains a simple fetcher to download a file from a remote URL and verify its
# SHA1 checksum.
#
# Usage: fetch.sh <remote URL> <SHA1 checksum>
set -e
# Pull the file from the remote URL
file=`basename $1`
echo "Downloading $1..."
wget -q $1
# Generate a desired checksum report and check against it
echo "$2 $file" > $file.sum
sha1sum -c $file.sum
rm $file.sum