deepcrayon
/
spack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity
spack/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

844 lines
25 KiB
YAML
Raw Blame History

stages: [ "generate", "build", "publish" ]
variables:
SPACK_DISABLE_LOCAL_CONFIG: "1"
SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"
default:
image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
# CI Platform-Arch
.cray_rhel_zen4:
variables:
SPACK_TARGET_PLATFORM: "cray-rhel"
SPACK_TARGET_ARCH: "zen4"
.cray_sles_zen4:
variables:
SPACK_TARGET_PLATFORM: "cray-sles"
SPACK_TARGET_ARCH: "zen4"
.darwin_x86_64:
variables:
SPACK_TARGET_PLATFORM: "darwin"
SPACK_TARGET_ARCH: "x86_64"
.darwin_aarch64:
variables:
SPACK_TARGET_PLATFORM: "darwin"
SPACK_TARGET_ARCH: "aarch64"
.linux_x86_64_v3:
variables:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "x86_64_v3"
.linux_skylake:
variables:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "skylake_avx512"
.linux_icelake:
variables:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "icelake"
.linux_neoverse_n1:
variables:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "neoverse_n1"
.linux_neoverse_v1:
variables:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "neoverse_v1"
.linux_aarch64:
variables:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "aarch64"
.linux_power:
variables:
SPACK_TARGET_PLATFORM: "linux"
SPACK_TARGET_ARCH: "ppc64le"
########################################
# Job templates
########################################
.base-job:
variables:
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
rules:
- if: $CI_COMMIT_REF_NAME == "develop"
# Pipelines on develop only rebuild what is missing from the mirror
when: always
variables:
SPACK_PIPELINE_TYPE: "spack_protected_branch"
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
SPACK_REQUIRE_SIGNING: "True"
AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
- if: $CI_COMMIT_REF_NAME =~ /^releases\/v.*/
# Pipelines on release branches always rebuild everything
when: always
variables:
SPACK_PIPELINE_TYPE: "spack_protected_branch"
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
SPACK_PRUNE_UNTOUCHED: "False"
SPACK_PRUNE_UP_TO_DATE: "False"
SPACK_REQUIRE_SIGNING: "True"
AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
- if: $CI_COMMIT_TAG =~ /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/ || $CI_COMMIT_TAG =~ /^v.*/
# Pipelines on tags (release or dev snapshots) only copy binaries from one mirror to another
when: always
variables:
SPACK_PIPELINE_TYPE: "spack_copy_only"
SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
- if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
# Pipelines on PR branches rebuild only what's missing, and do extra pruning
when: always
variables:
SPACK_PIPELINE_TYPE: "spack_pull_request"
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
SPACK_PRUNE_UNTOUCHED: "True"
SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"
.generate-common:
stage: generate
script:
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- . "./share/spack/setup-env.sh"
- spack --version
- cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- spack env activate --without-view .
- export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
- spack
--config-scope "${SPACK_CI_CONFIG_ROOT}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
--config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
${CI_STACK_CONFIG_SCOPES}
ci generate --check-index-only
--buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
after_script:
- cat /proc/loadavg || true
artifacts:
paths:
- "${CI_PROJECT_DIR}/jobs_scratch_dir"
variables:
KUBERNETES_CPU_REQUEST: 4000m
KUBERNETES_MEMORY_REQUEST: 16G
# avoid moving targets like branches and tags
SPACK_CONCRETIZER_REQUIRE_CHECKSUM: 1
SPACK_BACKTRACE: 1
interruptible: true
timeout: 60 minutes
retry:
max: 2
when:
- always
# Generate without tags for cases using external runners
.generate-base:
extends: [ ".base-job", ".generate-common" ]
.generate-x86_64:
extends: [ ".generate-base" ]
tags: ["spack", "public", "medium", "x86_64"]
.generate-aarch64:
extends: [ ".generate-base" ]
tags: ["spack", "public", "medium", "aarch64"]
.generate-deprecated:
extends: [ ".base-job" ]
stage: generate
script:
- uname -a || true
- grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
- nproc || true
- . "./share/spack/setup-env.sh"
- spack --version
- cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
- spack env activate --without-view .
- spack
ci generate --check-index-only
--buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
--artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
--output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
after_script:
- cat /proc/loadavg || true
artifacts:
paths:
- "${CI_PROJECT_DIR}/jobs_scratch_dir"
variables:
KUBERNETES_CPU_REQUEST: 4000m
KUBERNETES_MEMORY_REQUEST: 16G
interruptible: true
timeout: 60 minutes
retry:
max: 2
when:
- always
tags: ["spack", "public", "medium", "x86_64"]
.build:
extends: [ ".base-job" ]
stage: build
protected-publish:
# Copy binaries from stack-specific mirrors to a root mirror
stage: publish
only:
- /^develop$/
- /^releases\/v.*/
- /^v.*/
- /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/
image: "ghcr.io/spack/python-aws-bash:0.0.1"
tags: ["spack", "public", "medium", "aws", "x86_64"]
retry:
max: 2
when: ["runner_system_failure", "stuck_or_timeout_failure"]
variables:
SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
SPACK_PIPELINE_TYPE: "spack_protected_branch"
AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
KUBERNETES_CPU_REQUEST: 4000m
KUBERNETES_MEMORY_REQUEST: 16G
script:
- . "./share/spack/setup-env.sh"
- spack --version
- export COPY_SPECS_DIR=${CI_PROJECT_DIR}/jobs_scratch_dir/specs_to_copy
- spack buildcache sync --manifest-glob "${COPY_SPECS_DIR}/*.json"
- curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
- aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_BUILDCACHE}/build_cache/_pgp/spack-public-binary-key.pub"
- spack buildcache update-index --keys "${SPACK_COPY_BUILDCACHE}"
id_tokens:
GITLAB_OIDC_TOKEN:
aud: "protected_binary_mirror"
########################################
# TEMPLATE FOR ADDING ANOTHER PIPELINE
########################################
#
# First add a new spack.yml defining the pipeline to run in
# share/spack/gitlab/cloud_pipelines/stacks/my-super-cool-stack/spack.yaml
#
# Then add the following entries at the bottom of this file.
#
# Note that when extending other jobs, gitlab does not merge lists (nor
# does it merge dictionary values in the case of key conflicts). So lists
# and duplicated dictionary keys are simply overridden. For this reason,
# you should inlclude your custom definitions at the end of the of the
# extends list.
#
# Also note that if extending .base-job, the mirror url given in your
# spack.yaml should take the form:
#
# s3://spack-binaries/develop/${SPACK_CI_STACK_NAME}
#
########################################
# My Super Cool Pipeline
########################################
# .my-super-cool-stack:
# extends: [ ".linux_x86_64_v3" ]
# variables:
# SPACK_CI_STACK_NAME: my-super-cool-stack
# tags: [ "all", "tags", "your", "job", "needs"]
#
# my-super-cool-stack-generate:
# extends: [ ".generate", ".my-super-cool-stack" ]
#
# my-super-cool-stack-build:
# extends: [ ".build", ".my-super-cool-stack" ]
# trigger:
# include:
# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
# job: my-super-cool-stack-generate
# strategy: depend
# needs:
# - artifacts: True
# job: my-super-cool-stack-generate
########################################
# E4S pipeline
########################################
.e4s:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: e4s
e4s-generate:
extends: [ ".e4s", ".generate-x86_64"]
image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01
e4s-build:
extends: [ ".e4s", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: e4s-generate
strategy: depend
needs:
- artifacts: True
job: e4s-generate
########################################
# GPU Testing Pipeline
########################################
.gpu-tests:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: gpu-tests
gpu-tests-generate:
extends: [ ".gpu-tests", ".generate-x86_64"]
image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01
gpu-tests-build:
extends: [ ".gpu-tests", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: gpu-tests-generate
strategy: depend
needs:
- artifacts: True
job: gpu-tests-generate
########################################
# E4S OneAPI Pipeline
########################################
.e4s-oneapi:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: e4s-oneapi
e4s-oneapi-generate:
extends: [ ".e4s-oneapi", ".generate-x86_64"]
image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023.07.21
e4s-oneapi-build:
extends: [ ".e4s-oneapi", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: e4s-oneapi-generate
strategy: depend
needs:
- artifacts: True
job: e4s-oneapi-generate
########################################
# E4S on Power
########################################
.e4s-power-generate-tags-and-image:
image: { "name": "ecpe4s/ubuntu20.04-runner-ppc64le:2023-01-01", "entrypoint": [""] }
tags: ["spack", "public", "large", "ppc64le"]
.e4s-power:
extends: [".linux_power"]
variables:
SPACK_CI_STACK_NAME: e4s-power
e4s-power-generate:
extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]
e4s-power-build:
extends: [ ".e4s-power", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: e4s-power-generate
strategy: depend
needs:
- artifacts: True
job: e4s-power-generate
#########################################
# Build tests for different build-systems
#########################################
.build_systems:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: build_systems
build_systems-generate:
extends: [ ".build_systems", ".generate-x86_64"]
build_systems-build:
extends: [ ".build_systems", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: build_systems-generate
strategy: depend
needs:
- artifacts: True
job: build_systems-generate
#########################################
# RADIUSS
#########################################
.radiuss:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: radiuss
radiuss-generate:
extends: [ ".radiuss", ".generate-x86_64" ]
radiuss-build:
extends: [ ".radiuss", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: radiuss-generate
strategy: depend
needs:
- artifacts: True
job: radiuss-generate
########################################
# RADIUSS for AWS
########################################
.tags-x86_64_v4:
tags: ["spack", "public", "aws", "medium", "x86_64_v4"]
# Include this AFTER .*-generate in "extends" list
.radiuss-aws-overrides:
# This controls image for generate step; build step is controlled by spack.yaml
# Note that generator emits OS info for build so these should be the same.
image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }
.radiuss-aws:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: radiuss-aws
radiuss-aws-generate:
extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]
radiuss-aws-build:
extends: [ ".radiuss-aws", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: radiuss-aws-generate
strategy: depend
needs:
- artifacts: True
job: radiuss-aws-generate
# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
.radiuss-aws-aarch64:
extends: [ ".linux_aarch64" ]
variables:
SPACK_CI_STACK_NAME: radiuss-aws-aarch64
radiuss-aws-aarch64-generate:
extends: [ ".radiuss-aws-aarch64", ".generate-aarch64", ".radiuss-aws-overrides" ]
radiuss-aws-aarch64-build:
extends: [ ".radiuss-aws-aarch64", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: radiuss-aws-aarch64-generate
strategy: depend
needs:
- artifacts: True
job: radiuss-aws-aarch64-generate
########################################
# ECP Data & Vis SDK
########################################
.data-vis-sdk:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: data-vis-sdk
data-vis-sdk-generate:
extends: [ ".data-vis-sdk", ".generate-x86_64"]
image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01
data-vis-sdk-build:
extends: [ ".data-vis-sdk", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: data-vis-sdk-generate
strategy: depend
needs:
- artifacts: True
job: data-vis-sdk-generate
########################################
# AWS ISC Applications (x86_64)
########################################
# Call this AFTER .*-generate
.aws-isc-overrides:
# This controls image for generate step; build step is controlled by spack.yaml
# Note that generator emits OS info for build so these should be the same.
image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }
.aws-isc:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: aws-isc
aws-isc-generate:
extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ]
aws-isc-build:
extends: [ ".aws-isc", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: aws-isc-generate
strategy: depend
needs:
- artifacts: True
job: aws-isc-generate
# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)
.aws-isc-aarch64:
extends: [ ".linux_aarch64" ]
variables:
SPACK_CI_STACK_NAME: aws-isc-aarch64
aws-isc-aarch64-generate:
extends: [ ".aws-isc-aarch64", ".generate-aarch64", ".aws-isc-overrides" ]
aws-isc-aarch64-build:
extends: [ ".aws-isc-aarch64", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: aws-isc-aarch64-generate
strategy: depend
needs:
- artifacts: True
job: aws-isc-aarch64-generate
########################################
# Spack Tutorial
########################################
.tutorial:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: tutorial
tutorial-generate:
extends: [ ".tutorial", ".generate-x86_64"]
image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07
tutorial-build:
extends: [ ".tutorial", ".build" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: tutorial-generate
strategy: depend
needs:
- artifacts: True
job: tutorial-generate
#######################################
# Machine Learning - Linux x86_64 (CPU)
#######################################
.ml-linux-x86_64-cpu:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu
ml-linux-x86_64-cpu-generate:
extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-cpu-build:
extends: [ ".build", ".ml-linux-x86_64-cpu" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: ml-linux-x86_64-cpu-generate
strategy: depend
needs:
- artifacts: True
job: ml-linux-x86_64-cpu-generate
########################################
# Machine Learning - Linux x86_64 (CUDA)
########################################
.ml-linux-x86_64-cuda:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda
ml-linux-x86_64-cuda-generate:
extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-cuda-build:
extends: [ ".build", ".ml-linux-x86_64-cuda" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: ml-linux-x86_64-cuda-generate
strategy: depend
needs:
- artifacts: True
job: ml-linux-x86_64-cuda-generate
########################################
# Machine Learning - Linux x86_64 (ROCm)
########################################
.ml-linux-x86_64-rocm:
extends: [ ".linux_x86_64_v3" ]
variables:
SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm
ml-linux-x86_64-rocm-generate:
extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly
ml-linux-x86_64-rocm-build:
extends: [ ".build", ".ml-linux-x86_64-rocm" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: ml-linux-x86_64-rocm-generate
strategy: depend
needs:
- artifacts: True
job: ml-linux-x86_64-rocm-generate
########################################
# Machine Learning - Darwin aarch64 (MPS)
########################################
# No runners available right now
# .ml-darwin-aarch64-mps:
# extends: [".darwin_aarch64"]
# variables:
# SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
# ml-darwin-aarch64-mps-generate:
# tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
# extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]
# ml-darwin-aarch64-mps-build:
# extends: [ ".ml-darwin-aarch64-mps", ".build" ]
# trigger:
# include:
# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
# job: ml-darwin-aarch64-mps-generate
# strategy: depend
# needs:
# - artifacts: True
# job: ml-darwin-aarch64-mps-generate
########################################
# Deprecated CI testing
########################################
.deprecated-ci:
variables:
SPACK_CI_STACK_NAME: deprecated
deprecated-ci-generate:
extends: [ ".generate-deprecated", ".deprecated-ci" ]
deprecated-ci-build:
extends: [ ".build", ".deprecated-ci" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: deprecated-ci-generate
strategy: depend
needs:
- artifacts: True
job: deprecated-ci-generate
########################################
# AWS PCLUSTER
########################################
.aws-pcluster-generate-image:
image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2023-05-25", "entrypoint": [""] }
.aws-pcluster-generate:
before_script:
# Use gcc from local container buildcache
- - . "./share/spack/setup-env.sh"
- . /etc/profile.d/modules.sh
- spack buildcache rebuild-index /bootstrap/local-cache/
- spack mirror add local-cache /bootstrap/local-cache
- spack gpg trust /bootstrap/public-key
- cd "${CI_PROJECT_DIR}" && curl -sOL https://raw.githubusercontent.com/spack/spack-configs/main/AWS/parallelcluster/postinstall.sh
- sed -i -e "s/spack arch -t/echo ${SPACK_TARGET_ARCH}/g" postinstall.sh
- sed -i.bkp s/"spack install gcc"/"spack install --cache-only --reuse gcc"/ postinstall.sh
- diff postinstall.sh postinstall.sh.bkp || echo Done
- /bin/bash postinstall.sh -fg
- spack config --scope site add "packages:all:target:[${SPACK_TARGET_ARCH}]"
after_script:
- - mv "${CI_PROJECT_DIR}/postinstall.sh" "${CI_PROJECT_DIR}/jobs_scratch_dir/"
# Icelake (one pipeline per target)
.aws-pcluster-icelake:
variables:
SPACK_CI_STACK_NAME: aws-pcluster-icelake
# aws-pcluster-generate-icelake:
# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-icelake:
# extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ]
# trigger:
# include:
# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
# job: aws-pcluster-generate-icelake
# strategy: depend
# needs:
# - artifacts: True
# job: aws-pcluster-generate-icelake
# Skylake_avx512 (one pipeline per target)
.aws-pcluster-skylake:
variables:
SPACK_CI_STACK_NAME: aws-pcluster-skylake
# aws-pcluster-generate-skylake:
# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-skylake:
# extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ]
# trigger:
# include:
# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
# job: aws-pcluster-generate-skylake
# strategy: depend
# needs:
# - artifacts: True
# job: aws-pcluster-generate-skylake
# Neoverse_n1 (one pipeline per target)
.aws-pcluster-neoverse_n1:
variables:
SPACK_CI_STACK_NAME: aws-pcluster-neoverse_n1
# aws-pcluster-generate-neoverse_n1:
# extends: [ ".linux_neoverse_n1", ".aws-pcluster-neoverse_n1", ".generate-aarch64", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-neoverse_n1:
# extends: [ ".linux_neoverse_n1", ".aws-pcluster-neoverse_n1", ".build" ]
# trigger:
# include:
# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
# job: aws-pcluster-generate-neoverse_n1
# strategy: depend
# needs:
# - artifacts: True
# job: aws-pcluster-generate-neoverse_n1
# Neoverse_v1 (one pipeline per target)
.aws-pcluster-neoverse_v1:
variables:
SPACK_CI_STACK_NAME: aws-pcluster-neoverse_v1
# aws-pcluster-generate-neoverse_v1:
# extends: [ ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".generate-aarch64", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
# aws-pcluster-build-neoverse_v1:
# extends: [ ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".build" ]
# trigger:
# include:
# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
# job: aws-pcluster-generate-neoverse_v1
# strategy: depend
# needs:
# - artifacts: True
# job: aws-pcluster-generate-neoverse_v1
# Cray definitions
.generate-cray:
extends: [ ".generate-common", ".base-job" ]
before_script:
- echo $PATH
- module avail
- module list
.generate-cray-rhel:
tags: [ "cray-rhel-zen4", "public" ]
extends: [ ".generate-cray" ]
.generate-cray-sles:
tags: [ "cray-sles-zen4", "public" ]
extends: [ ".generate-cray" ]
#######################################
# E4S - Cray RHEL
#######################################
.e4s-cray-rhel:
extends: [ ".cray_rhel_zen4" ]
variables:
SPACK_CI_STACK_NAME: e4s-cray-rhel
e4s-cray-rhel-generate:
extends: [ ".generate-cray-rhel", ".e4s-cray-rhel" ]
e4s-cray-rhel-build:
extends: [ ".build", ".e4s-cray-rhel" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: e4s-cray-rhel-generate
strategy: depend
needs:
- artifacts: True
job: e4s-cray-rhel-generate
#######################################
# E4S - Cray SLES
#######################################
.e4s-cray-sles:
extends: [ ".cray_sles_zen4" ]
variables:
SPACK_CI_STACK_NAME: e4s-cray-sles
e4s-cray-sles-generate:
extends: [ ".generate-cray-sles", ".e4s-cray-sles" ]
e4s-cray-sles-build:
extends: [ ".build", ".e4s-cray-sles" ]
trigger:
include:
- artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
job: e4s-cray-sles-generate
strategy: depend
needs:
- artifacts: True
job: e4s-cray-sles-generate
Reference in New Issue View Git Blame Copy Permalink