From a7c454648d95a6dd70cb70bdefb6863cb9cc5391 Mon Sep 17 00:00:00 2001 From: Jan Kieseler Date: Tue, 7 May 2024 13:58:10 +0200 Subject: [PATCH] updated build pipeline --- .gitlab-ci.yml | 50 +--------------------------------- docker/Dockerfile | 18 ++++++++++--- docker/build.sh | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 52 deletions(-) create mode 100755 docker/build.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 64b372d..bc63beb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,49 +1 @@ -# This file is a template, and might need editing before it works on your project. -# This is a sample GitLab CI/CD configuration file that should run without any modifications. -# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, -# it uses echo commands to simulate the pipeline execution. -# -# A pipeline is composed of independent jobs that run scripts, grouped into stages. -# Stages run in sequential order, but jobs within stages run in parallel. -# -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages -# -# You can copy and paste this template into a new `.gitlab-ci.yml` file. -# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. -# -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - -stages: # List of stages for jobs, and their order of execution - - build - - test - - deploy - -build-job: # This job runs in the build stage, which runs first. - stage: build - script: - - echo "Compiling the code..." - - echo "Compile complete." - -unit-test-job: # This job runs in the test stage. - stage: test # It only starts when the job in the build stage completes successfully. - script: - - echo "Running unit tests... This will take about 60 seconds." - - sleep 60 - - echo "Code coverage is 90%" - -lint-test-job: # This job also runs in the test stage. - stage: test # It can run at the same time as unit-test-job (in parallel). - script: - - echo "Linting code... This will take about 10 seconds." - - sleep 10 - - echo "No lint issues found." - -deploy-job: # This job runs in the deploy stage. - stage: deploy # It only runs when *both* jobs in the test stage complete successfully. - environment: production - script: - - echo "Deploying application..." - - echo "Application successfully deployed." +# empty \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d74fe0..fbec515 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -81,10 +81,22 @@ RUN apt-get install -y tk # for dawn RUN python3 -m pip install jupyterlab jupyterhub==3.1.1 batchspawner RUN ln -s /usr/bin/python3 /usr/bin/python -#finally the package -ARG INCUBATOR_VER + +RUN python3 -m pip install torch-scatter -f https://data.pyg.org/whl/torch-2.1.0+cu118.html + +#finally the package; use the commit hash to check out the correct version and only get them here, such that the rest can be cached +ARG BUILD_DATE +LABEL org.label-schema.build-date=$BUILD_DATE +ARG COMMIT +ARG USER + +# copy minicalosim and checkout the correct commit ADD minicalosim /root/minicalosim -RUN cd && cd minicalosim && mkdir -p build && cd build && cmake ../ && make -j4 &&\ + +RUN cd /root/minicalosim && git checkout $COMMIT && \ + mkdir -p build && cd build && rm -rf * && cmake ../ && make -j4 &&\ cp minicalo* ../bind/G4Calo.py /usr/local/lib/python3.8/dist-packages/ +# clean up +RUN rm -rf /root/minicalosim diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 0000000..22aa4b0 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,69 @@ +#!/usr/bin/bash + +# a simple parser for the args and --no-cache +# if no commit is given, the latest commit is used +# if --no-cache is given, the --no-cache flag is passed to the docker build command +# if only --no-cache is given, the latest commit is used + +# check if commit is given as argument, if not use the latest commit +# check if an argument is given at all + + +if [ -n "$1" ]; then + #check if the first arg is --no-cache + if [ "$1" == "--no-cache" ]; then + FORCE_NO_CACHE="--no-cache" + COMMIT=$(git rev-parse --short HEAD) + else + COMMIT=$1 + #check if this is a valid commit + if ! git cat-file -e $COMMIT^{commit} 2>/dev/null; then + echo "ERROR: Commit $COMMIT not found in local repository." + exit 1 + fi + fi +else #default to last commit + COMMIT=$(git rev-parse --short HEAD) +fi + + +# COMMIT is a valid commit at this point. If COMMIT is the last commit and changes are not commited, the user is warned. + +#check if the commit is the last commit +if [ "$COMMIT" == "$(git rev-parse --short HEAD)" ]; then + # check if we have uncommitted changes and if so warn the user. + if [ -n "$(git status --porcelain)" ]; then + echo "WARNING: You are building the latest commit ${COMMIT}, but you have uncommitted changes." + echo " This means that the image will not be reproducible." + echo " If you want to build the latest commit, please commit your changes first." + read -p "Do you want to continue anyway? [y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + fi +fi + +# check if the commit has been pushed +REMOTE_COMMIT=$(git ls-remote https://gitlab.etp.kit.edu/jkiesele/minicalosim.git | grep $COMMIT) +if [ -z "$REMOTE_COMMIT" ]; then + echo "ERROR: Commit $COMMIT not found in remote repository." + exit 1 +fi + +# check if no-cache is given as second argument +if [ "$2" == "--no-cache" ]; then + FORCE_NO_CACHE="--no-cache" +fi + +#switch to directory this script is located in +cd "$(dirname "$0")" +cd ../../ #switch to the root directory of the project +echo "Building minicalosim:$COMMIT" + +#print the current working directory +echo "Current working directory: $(pwd)" + +docker build $FORCE_NO_CACHE -t jkiesele/minicalosim:$COMMIT --build-arg USER=$USER --build-arg BUILD_DATE="$(date)" --build-arg COMMIT=$COMMIT -f minicalosim/docker/Dockerfile . + +echo "sucessfully built container jkiesele/minicalosim:${COMMIT}" \ No newline at end of file