1436 lines
55 KiB
YAML
1436 lines
55 KiB
YAML
# WARNING: DO NOT EDIT THIS FILE DIRECTLY!!!
|
|
# See the README.md in this directory.
|
|
|
|
# IMPORTANT: To update Docker image version, please follow
|
|
# the instructions at
|
|
# https://github.com/pytorch/pytorch/wiki/Docker-image-build-on-CircleCI
|
|
|
|
version: 2.1
|
|
|
|
parameters:
|
|
run_binary_tests:
|
|
type: boolean
|
|
default: false
|
|
run_build:
|
|
type: boolean
|
|
default: true
|
|
run_master_build:
|
|
type: boolean
|
|
default: false
|
|
run_slow_gradcheck_build:
|
|
type: boolean
|
|
default: false
|
|
|
|
executors:
|
|
windows-with-nvidia-gpu:
|
|
machine:
|
|
resource_class: windows.gpu.nvidia.medium
|
|
image: windows-server-2019-nvidia:previous
|
|
shell: bash.exe
|
|
|
|
windows-xlarge-cpu-with-nvidia-cuda:
|
|
machine:
|
|
resource_class: windows.xlarge
|
|
image: windows-server-2019-vs2019:stable
|
|
shell: bash.exe
|
|
|
|
windows-medium-cpu-with-nvidia-cuda:
|
|
machine:
|
|
resource_class: windows.medium
|
|
image: windows-server-2019-vs2019:stable
|
|
shell: bash.exe
|
|
commands:
|
|
|
|
calculate_docker_image_tag:
|
|
description: "Calculates the docker image tag"
|
|
steps:
|
|
- run:
|
|
name: "Calculate docker image hash"
|
|
command: |
|
|
DOCKER_TAG=$(git rev-parse HEAD:.ci/docker)
|
|
echo "DOCKER_TAG=${DOCKER_TAG}" >> "${BASH_ENV}"
|
|
|
|
designate_upload_channel:
|
|
description: "inserts the correct upload channel into ${BASH_ENV}"
|
|
steps:
|
|
- run:
|
|
name: adding UPLOAD_CHANNEL to BASH_ENV
|
|
command: |
|
|
our_upload_channel=nightly
|
|
# On tags upload to test instead
|
|
if [[ -n "${CIRCLE_TAG}" ]]; then
|
|
our_upload_channel=test
|
|
fi
|
|
echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV}
|
|
|
|
# This system setup script is meant to run before the CI-related scripts, e.g.,
|
|
# installing Git client, checking out code, setting up CI env, and
|
|
# building/testing.
|
|
setup_linux_system_environment:
|
|
steps:
|
|
- run:
|
|
name: Set Up System Environment
|
|
no_output_timeout: "1h"
|
|
command: .circleci/scripts/setup_linux_system_environment.sh
|
|
|
|
setup_ci_environment:
|
|
steps:
|
|
- run:
|
|
name: Set Up CI Environment After attach_workspace
|
|
no_output_timeout: "1h"
|
|
command: .circleci/scripts/setup_ci_environment.sh
|
|
|
|
brew_update:
|
|
description: "Update Homebrew and install base formulae"
|
|
steps:
|
|
- run:
|
|
name: Update Homebrew
|
|
no_output_timeout: "10m"
|
|
command: |
|
|
set -ex
|
|
|
|
# Update repositories manually.
|
|
# Running `brew update` produces a comparison between the
|
|
# current checkout and the updated checkout, which takes a
|
|
# very long time because the existing checkout is 2y old.
|
|
for path in $(find /usr/local/Homebrew -type d -name .git)
|
|
do
|
|
cd $path/..
|
|
git fetch --depth=1 origin
|
|
git reset --hard origin/master
|
|
done
|
|
|
|
export HOMEBREW_NO_AUTO_UPDATE=1
|
|
|
|
# Install expect and moreutils so that we can call `unbuffer` and `ts`.
|
|
# moreutils installs a `parallel` executable by default, which conflicts
|
|
# with the executable from the GNU `parallel`, so we must unlink GNU
|
|
# `parallel` first, and relink it afterwards.
|
|
brew unlink parallel
|
|
brew install moreutils
|
|
brew link parallel --overwrite
|
|
brew install expect
|
|
|
|
brew_install:
|
|
description: "Install Homebrew formulae"
|
|
parameters:
|
|
formulae:
|
|
type: string
|
|
default: ""
|
|
steps:
|
|
- run:
|
|
name: Install << parameters.formulae >>
|
|
no_output_timeout: "10m"
|
|
command: |
|
|
set -ex
|
|
export HOMEBREW_NO_AUTO_UPDATE=1
|
|
brew install << parameters.formulae >>
|
|
|
|
run_brew_for_macos_build:
|
|
steps:
|
|
- brew_update
|
|
- brew_install:
|
|
formulae: libomp
|
|
|
|
run_brew_for_ios_build:
|
|
steps:
|
|
- brew_update
|
|
- brew_install:
|
|
formulae: libtool
|
|
|
|
optional_merge_target_branch:
|
|
steps:
|
|
- run:
|
|
name: (Optional) Merge target branch
|
|
no_output_timeout: "10m"
|
|
command: |
|
|
if [[ -n "$CIRCLE_PULL_REQUEST" && "$CIRCLE_BRANCH" != "nightly" ]]; then
|
|
PR_NUM=$(basename $CIRCLE_PULL_REQUEST)
|
|
CIRCLE_PR_BASE_BRANCH=$(curl -s https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$PR_NUM | jq -r '.base.ref')
|
|
if [[ "${BUILD_ENVIRONMENT}" == *"xla"* || "${BUILD_ENVIRONMENT}" == *"gcc5"* ]] ; then
|
|
set -x
|
|
git config --global user.email "circleci.ossci@gmail.com"
|
|
git config --global user.name "CircleCI"
|
|
git config remote.origin.url https://github.com/pytorch/pytorch.git
|
|
git config --add remote.origin.fetch +refs/heads/master:refs/remotes/origin/master
|
|
git fetch --tags --progress https://github.com/pytorch/pytorch.git +refs/heads/master:refs/remotes/origin/master --depth=100 --quiet
|
|
# PRs generated from ghstack has format CIRCLE_PR_BASE_BRANCH=gh/xxx/1234/base
|
|
if [[ "${CIRCLE_PR_BASE_BRANCH}" == "gh/"* ]]; then
|
|
CIRCLE_PR_BASE_BRANCH=master
|
|
fi
|
|
export GIT_MERGE_TARGET=`git log -n 1 --pretty=format:"%H" origin/$CIRCLE_PR_BASE_BRANCH`
|
|
echo "GIT_MERGE_TARGET: " ${GIT_MERGE_TARGET}
|
|
export GIT_COMMIT=${CIRCLE_SHA1}
|
|
echo "GIT_COMMIT: " ${GIT_COMMIT}
|
|
git checkout -f ${GIT_COMMIT}
|
|
git reset --hard ${GIT_COMMIT}
|
|
git merge --allow-unrelated-histories --no-edit --no-ff ${GIT_MERGE_TARGET}
|
|
echo "Merged $CIRCLE_PR_BASE_BRANCH branch before building in environment $BUILD_ENVIRONMENT"
|
|
set +x
|
|
else
|
|
echo "No need to merge with $CIRCLE_PR_BASE_BRANCH, skipping..."
|
|
fi
|
|
else
|
|
echo "This is not a pull request, skipping..."
|
|
fi
|
|
|
|
##############################################################################
|
|
# Binary build (nightlies nightly build) defaults
|
|
# The binary builds use the docker executor b/c at time of writing the machine
|
|
# executor is limited to only two cores and is painfully slow (4.5+ hours per
|
|
# GPU build). But the docker executor cannot be run with --runtime=nvidia, and
|
|
# so the binary test/upload jobs must run on a machine executor. The package
|
|
# built in the build job is persisted to the workspace, which the test jobs
|
|
# expect. The test jobs just run a few quick smoke tests (very similar to the
|
|
# second-round-user-facing smoke tests above) and then upload the binaries to
|
|
# their final locations. The upload part requires credentials that should only
|
|
# be available to org-members.
|
|
#
|
|
# binary_checkout MUST be run before other commands here. This is because the
|
|
# other commands are written in .circleci/scripts/*.sh , so the pytorch source
|
|
# code must be downloaded on the machine before they can be run. We cannot
|
|
# inline all the code into this file, since that would cause the yaml size to
|
|
# explode past 4 MB (all the code in the command section is just copy-pasted to
|
|
# everywhere in the .circleci/config.yml file where it appears).
|
|
##############################################################################
|
|
|
|
# Checks out the Pytorch and Builder repos (always both of them), and places
|
|
# them in the right place depending on what executor we're running on. We curl
|
|
# our .sh file from the interweb to avoid yaml size bloat. Note that many jobs
|
|
# do not need both the pytorch and builder repos, so this is a little wasteful
|
|
# (smoke tests and upload jobs do not need the pytorch repo).
|
|
binary_checkout: &binary_checkout
|
|
name: Checkout pytorch/builder repo
|
|
no_output_timeout: "30m"
|
|
command: .circleci/scripts/binary_checkout.sh
|
|
|
|
# Parses circleci arguments in a consistent way, essentially routing to the
|
|
# correct pythonXgccXcudaXos build we want
|
|
binary_populate_env: &binary_populate_env
|
|
name: Set up binary env variables
|
|
command: .circleci/scripts/binary_populate_env.sh
|
|
|
|
binary_install_miniconda: &binary_install_miniconda
|
|
name: Install miniconda
|
|
no_output_timeout: "1h"
|
|
command: .circleci/scripts/binary_install_miniconda.sh
|
|
|
|
# This section is used in the binary_test and smoke_test jobs. It expects
|
|
# 'binary_populate_env' to have populated /home/circleci/project/env and it
|
|
# expects another section to populate /home/circleci/project/ci_test_script.sh
|
|
# with the code to run in the docker
|
|
binary_run_in_docker: &binary_run_in_docker
|
|
name: Run in docker
|
|
# This step only runs on circleci linux machine executors that themselves
|
|
# need to start docker images
|
|
command: .circleci/scripts/binary_run_in_docker.sh
|
|
##############################################################################
|
|
# Build parameters
|
|
##############################################################################
|
|
pytorch_params: &pytorch_params
|
|
parameters:
|
|
build_environment:
|
|
type: string
|
|
default: ""
|
|
docker_image:
|
|
type: string
|
|
default: ""
|
|
resource_class:
|
|
type: string
|
|
default: "large"
|
|
use_cuda_docker_runtime:
|
|
type: string
|
|
default: ""
|
|
build_only:
|
|
type: string
|
|
default: ""
|
|
ci_master:
|
|
type: string
|
|
default: ""
|
|
environment:
|
|
BUILD_ENVIRONMENT: << parameters.build_environment >>
|
|
DOCKER_IMAGE: << parameters.docker_image >>
|
|
USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
|
|
BUILD_ONLY: << parameters.build_only >>
|
|
CI_MASTER: << pipeline.parameters.run_master_build >>
|
|
resource_class: << parameters.resource_class >>
|
|
|
|
pytorch_ios_params: &pytorch_ios_params
|
|
parameters:
|
|
build_environment:
|
|
type: string
|
|
default: ""
|
|
ios_arch:
|
|
type: string
|
|
default: ""
|
|
ios_platform:
|
|
type: string
|
|
default: ""
|
|
op_list:
|
|
type: string
|
|
default: ""
|
|
use_metal:
|
|
type: string
|
|
default: "0"
|
|
lite_interpreter:
|
|
type: string
|
|
default: "1"
|
|
use_coreml:
|
|
type: string
|
|
default: "0"
|
|
environment:
|
|
BUILD_ENVIRONMENT: << parameters.build_environment >>
|
|
IOS_ARCH: << parameters.ios_arch >>
|
|
IOS_PLATFORM: << parameters.ios_platform >>
|
|
SELECTED_OP_LIST: << parameters.op_list >>
|
|
USE_PYTORCH_METAL: << parameters.use_metal >>
|
|
BUILD_LITE_INTERPRETER: << parameters.lite_interpreter >>
|
|
USE_COREML_DELEGATE: << parameters.use_coreml >>
|
|
|
|
pytorch_windows_params: &pytorch_windows_params
|
|
parameters:
|
|
executor:
|
|
type: string
|
|
default: "windows-xlarge-cpu-with-nvidia-cuda"
|
|
build_environment:
|
|
type: string
|
|
default: ""
|
|
test_name:
|
|
type: string
|
|
default: ""
|
|
cuda_version:
|
|
type: string
|
|
default: "10.1"
|
|
python_version:
|
|
type: string
|
|
default: "3.8"
|
|
vs_version:
|
|
type: string
|
|
default: "16.8.6"
|
|
vc_version:
|
|
type: string
|
|
default: "14.16"
|
|
vc_year:
|
|
type: string
|
|
default: "2019"
|
|
vc_product:
|
|
type: string
|
|
default: "BuildTools"
|
|
use_cuda:
|
|
type: string
|
|
default: ""
|
|
environment:
|
|
BUILD_ENVIRONMENT: <<parameters.build_environment>>
|
|
SCCACHE_BUCKET: "ossci-compiler-cache"
|
|
CUDA_VERSION: <<parameters.cuda_version>>
|
|
PYTHON_VERSION: <<parameters.python_version>>
|
|
VS_VERSION: <<parameters.vs_version>>
|
|
VC_VERSION: <<parameters.vc_version>>
|
|
VC_YEAR: <<parameters.vc_year>>
|
|
VC_PRODUCT: <<parameters.vc_product>>
|
|
USE_CUDA: <<parameters.use_cuda>>
|
|
TORCH_CUDA_ARCH_LIST: "5.2 7.5"
|
|
JOB_BASE_NAME: <<parameters.test_name>>
|
|
JOB_EXECUTOR: <<parameters.executor>>
|
|
binary_linux_build_params: &binary_linux_build_params
|
|
parameters:
|
|
build_environment:
|
|
type: string
|
|
default: ""
|
|
docker_image:
|
|
type: string
|
|
default: ""
|
|
libtorch_variant:
|
|
type: string
|
|
default: ""
|
|
resource_class:
|
|
type: string
|
|
default: "2xlarge+"
|
|
environment:
|
|
BUILD_ENVIRONMENT: << parameters.build_environment >>
|
|
LIBTORCH_VARIANT: << parameters.libtorch_variant >>
|
|
ANACONDA_USER: pytorch
|
|
resource_class: << parameters.resource_class >>
|
|
docker:
|
|
- image: << parameters.docker_image >>
|
|
|
|
binary_linux_test_upload_params: &binary_linux_test_upload_params
|
|
parameters:
|
|
build_environment:
|
|
type: string
|
|
default: ""
|
|
docker_image:
|
|
type: string
|
|
default: ""
|
|
libtorch_variant:
|
|
type: string
|
|
default: ""
|
|
resource_class:
|
|
type: string
|
|
default: "medium"
|
|
use_cuda_docker_runtime:
|
|
type: string
|
|
default: ""
|
|
environment:
|
|
BUILD_ENVIRONMENT: << parameters.build_environment >>
|
|
DOCKER_IMAGE: << parameters.docker_image >>
|
|
USE_CUDA_DOCKER_RUNTIME: << parameters.use_cuda_docker_runtime >>
|
|
LIBTORCH_VARIANT: << parameters.libtorch_variant >>
|
|
resource_class: << parameters.resource_class >>
|
|
|
|
binary_mac_params: &binary_mac_params
|
|
parameters:
|
|
build_environment:
|
|
type: string
|
|
default: ""
|
|
environment:
|
|
BUILD_ENVIRONMENT: << parameters.build_environment >>
|
|
|
|
binary_windows_params: &binary_windows_params
|
|
parameters:
|
|
build_environment:
|
|
type: string
|
|
default: ""
|
|
executor:
|
|
type: string
|
|
default: "windows-xlarge-cpu-with-nvidia-cuda"
|
|
environment:
|
|
BUILD_ENVIRONMENT: << parameters.build_environment >>
|
|
JOB_EXECUTOR: <<parameters.executor>>
|
|
##############################################################################
|
|
# Job specs
|
|
##############################################################################
|
|
jobs:
|
|
binary_ios_build:
|
|
<<: *pytorch_ios_params
|
|
macos:
|
|
xcode: "12.5.1"
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/workspace
|
|
- checkout
|
|
- run_brew_for_ios_build
|
|
- run:
|
|
name: Build
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
script="/Users/distiller/project/.circleci/scripts/binary_ios_build.sh"
|
|
cat "$script"
|
|
source "$script"
|
|
- run:
|
|
name: Test
|
|
no_output_timeout: "30m"
|
|
command: |
|
|
script="/Users/distiller/project/.circleci/scripts/binary_ios_test.sh"
|
|
cat "$script"
|
|
source "$script"
|
|
- persist_to_workspace:
|
|
root: /Users/distiller/workspace/
|
|
paths: ios
|
|
|
|
binary_ios_upload:
|
|
<<: *pytorch_ios_params
|
|
macos:
|
|
xcode: "12.5.1"
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/workspace
|
|
- checkout
|
|
- run_brew_for_ios_build
|
|
- run:
|
|
name: Upload
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
script="/Users/distiller/project/.circleci/scripts/binary_ios_upload.sh"
|
|
cat "$script"
|
|
source "$script"
|
|
|
|
anaconda_prune:
|
|
parameters:
|
|
packages:
|
|
type: string
|
|
description: "What packages are we pruning? (quoted, space-separated string. eg. 'pytorch', 'torchvision torchaudio', etc.)"
|
|
default: "pytorch"
|
|
channel:
|
|
type: string
|
|
description: "What channel are we pruning? (eq. pytorch-nightly)"
|
|
default: "pytorch-nightly"
|
|
docker:
|
|
- image: continuumio/miniconda3
|
|
environment:
|
|
- PACKAGES: "<< parameters.packages >>"
|
|
- CHANNEL: "<< parameters.channel >>"
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install dependencies
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
conda install -yq anaconda-client
|
|
- run:
|
|
name: Prune packages
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
ANACONDA_API_TOKEN="${CONDA_PYTORCHBOT_TOKEN}" \
|
|
scripts/release/anaconda-prune/run.sh
|
|
pytorch_doc_push:
|
|
resource_class: medium
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
parameters:
|
|
branch:
|
|
type: string
|
|
default: "main"
|
|
steps:
|
|
- attach_workspace:
|
|
at: /tmp/workspace
|
|
- run:
|
|
name: Generate netrc
|
|
command: |
|
|
# set credentials for https pushing
|
|
cat > ~/.netrc \<<DONE
|
|
machine github.com
|
|
login pytorchbot
|
|
password ${GITHUB_PYTORCHBOT_TOKEN}
|
|
DONE
|
|
- run:
|
|
name: Docs push
|
|
command: |
|
|
pushd /tmp/workspace
|
|
git push -u origin "<< parameters.branch >>"
|
|
|
|
pytorch_macos_10_15_py3_build:
|
|
environment:
|
|
BUILD_ENVIRONMENT: pytorch-macos-10.15-py3-arm64-build
|
|
macos:
|
|
xcode: "12.3.0"
|
|
steps:
|
|
- checkout
|
|
- run_brew_for_macos_build
|
|
- run:
|
|
name: Build
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
export CROSS_COMPILE_ARM64=1
|
|
export JOB_BASE_NAME=$CIRCLE_JOB
|
|
|
|
# Install sccache
|
|
sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
|
|
sudo chmod +x /usr/local/bin/sccache
|
|
export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
|
|
|
|
# This IAM user allows write access to S3 bucket for sccache
|
|
set +x
|
|
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
|
|
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
|
|
set -x
|
|
|
|
chmod a+x .ci/pytorch/macos-build.sh
|
|
unbuffer .ci/pytorch/macos-build.sh 2>&1 | ts
|
|
|
|
- persist_to_workspace:
|
|
root: /Users/distiller/workspace/
|
|
paths:
|
|
- miniconda3
|
|
- store_artifacts:
|
|
path: /Users/distiller/project/dist
|
|
|
|
pytorch_macos_10_13_py3_build:
|
|
environment:
|
|
BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-build
|
|
macos:
|
|
xcode: "12.0"
|
|
steps:
|
|
- checkout
|
|
- run_brew_for_macos_build
|
|
- run:
|
|
name: Build
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
export JOB_BASE_NAME=$CIRCLE_JOB
|
|
|
|
# Install sccache
|
|
sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
|
|
sudo chmod +x /usr/local/bin/sccache
|
|
export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
|
|
|
|
# This IAM user allows write access to S3 bucket for sccache
|
|
set +x
|
|
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
|
|
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
|
|
set -x
|
|
|
|
chmod a+x .ci/pytorch/macos-build.sh
|
|
unbuffer .ci/pytorch/macos-build.sh 2>&1 | ts
|
|
|
|
- persist_to_workspace:
|
|
root: /Users/distiller/workspace/
|
|
paths:
|
|
- miniconda3
|
|
|
|
mac_build:
|
|
parameters:
|
|
build-environment:
|
|
type: string
|
|
description: Top-level label for what's being built/tested.
|
|
xcode-version:
|
|
type: string
|
|
default: "13.3.1"
|
|
description: What xcode version to build with.
|
|
build-generates-artifacts:
|
|
type: boolean
|
|
default: true
|
|
description: if the build generates build artifacts
|
|
python-version:
|
|
type: string
|
|
default: "3.8"
|
|
macos:
|
|
xcode: << parameters.xcode-version >>
|
|
resource_class: medium
|
|
environment:
|
|
BUILD_ENVIRONMENT: << parameters.build-environment >>
|
|
AWS_REGION: us-east-1
|
|
steps:
|
|
|
|
- checkout
|
|
- run_brew_for_macos_build
|
|
|
|
- run:
|
|
name: Install sccache
|
|
command: |
|
|
sudo curl --retry 3 https://s3.amazonaws.com/ossci-macos/sccache_v2.15 --output /usr/local/bin/sccache
|
|
sudo chmod +x /usr/local/bin/sccache
|
|
echo "export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2" >> "${BASH_ENV}"
|
|
echo "export SCCACHE_S3_KEY_PREFIX=${GITHUB_WORKFLOW}" >> "${BASH_ENV}"
|
|
|
|
set +x
|
|
echo "export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}" >> "${BASH_ENV}"
|
|
echo "export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}" >> "${BASH_ENV}"
|
|
set -x
|
|
|
|
- run:
|
|
name: Get workflow job id
|
|
command: |
|
|
echo "export OUR_GITHUB_JOB_ID=${CIRCLE_WORKFLOW_JOB_ID}" >> "${BASH_ENV}"
|
|
|
|
- run:
|
|
name: Build
|
|
command: |
|
|
set -x
|
|
|
|
git submodule sync
|
|
git submodule update --init --recursive --depth 1 --jobs 0
|
|
|
|
export PATH="/usr/local/bin:$PATH"
|
|
export WORKSPACE_DIR="${HOME}/workspace"
|
|
mkdir -p "${WORKSPACE_DIR}"
|
|
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py38_4.12.0-MacOSX-x86_64.sh"
|
|
if [ << parameters.python-version >> == 3.9.12 ]; then
|
|
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-MacOSX-x86_64.sh"
|
|
fi
|
|
|
|
# If a local installation of conda doesn't exist, we download and install conda
|
|
if [ ! -d "${WORKSPACE_DIR}/miniconda3" ]; then
|
|
mkdir -p "${WORKSPACE_DIR}"
|
|
curl --retry 3 ${MINICONDA_URL} -o "${WORKSPACE_DIR}"/miniconda3.sh
|
|
bash "${WORKSPACE_DIR}"/miniconda3.sh -b -p "${WORKSPACE_DIR}"/miniconda3
|
|
fi
|
|
export PATH="${WORKSPACE_DIR}/miniconda3/bin:$PATH"
|
|
# shellcheck disable=SC1091
|
|
source "${WORKSPACE_DIR}"/miniconda3/bin/activate
|
|
|
|
brew link --force libomp
|
|
|
|
echo "export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname "$(which conda)")/../"}" >> "${BASH_ENV}"
|
|
.ci/pytorch/macos-build.sh
|
|
|
|
- when:
|
|
condition: << parameters.build-generates-artifacts >>
|
|
steps:
|
|
- run:
|
|
name: Archive artifacts into zip
|
|
command: |
|
|
zip -1 -r artifacts.zip dist/ build/.ninja_log build/compile_commands.json .pytorch-test-times.json .pytorch-test-file-ratings.json
|
|
cp artifacts.zip /Users/distiller/workspace
|
|
|
|
- persist_to_workspace:
|
|
root: /Users/distiller/workspace/
|
|
paths:
|
|
- miniconda3
|
|
- artifacts.zip
|
|
|
|
- store_artifacts:
|
|
path: /Users/distiller/project/artifacts.zip
|
|
|
|
mac_test:
|
|
parameters:
|
|
build-environment:
|
|
type: string
|
|
shard-number:
|
|
type: string
|
|
num-test-shards:
|
|
type: string
|
|
xcode-version:
|
|
type: string
|
|
test-config:
|
|
type: string
|
|
default: 'default'
|
|
|
|
macos:
|
|
xcode: << parameters.xcode-version >>
|
|
environment:
|
|
GIT_DEFAULT_BRANCH: 'master'
|
|
BUILD_ENVIRONMENT: << parameters.build-environment >>
|
|
TEST_CONFIG: << parameters.test-config >>
|
|
SHARD_NUMBER: << parameters.shard-number >>
|
|
NUM_TEST_SHARDS: << parameters.num-test-shards >>
|
|
PYTORCH_RETRY_TEST_CASES: 1
|
|
PYTORCH_OVERRIDE_FLAKY_SIGNAL: 1
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/workspace
|
|
- run_brew_for_macos_build
|
|
- run:
|
|
name: Test
|
|
no_output_timeout: "2h"
|
|
command: |
|
|
set -x
|
|
|
|
git submodule sync --recursive
|
|
git submodule update --init --recursive
|
|
|
|
mv ~/workspace/artifacts.zip .
|
|
unzip artifacts.zip
|
|
|
|
export IN_CI=1
|
|
|
|
COMMIT_MESSAGES=$(git cherry -v "origin/${GIT_DEFAULT_BRANCH:-master}")
|
|
|
|
export PATH="/usr/local/bin:$PATH"
|
|
export WORKSPACE_DIR="${HOME}/workspace"
|
|
mkdir -p "${WORKSPACE_DIR}"
|
|
|
|
export PATH="${WORKSPACE_DIR}/miniconda3/bin:$PATH"
|
|
source "${WORKSPACE_DIR}"/miniconda3/bin/activate
|
|
|
|
# sanitize the input commit message and PR body here:
|
|
|
|
# trim all new lines from commit messages to avoid issues with batch environment
|
|
# variable copying. see https://github.com/pytorch/pytorch/pull/80043#issuecomment-1167796028
|
|
COMMIT_MESSAGES="${COMMIT_MESSAGES//[$'\n\r']}"
|
|
|
|
# then trim all special characters like single and double quotes to avoid unescaped inputs to
|
|
# wreak havoc internally
|
|
export COMMIT_MESSAGES="${COMMIT_MESSAGES//[\'\"]}"
|
|
|
|
python3 -mpip install dist/*.whl
|
|
.ci/pytorch/macos-test.sh
|
|
- run:
|
|
name: Copy files for uploading test stats
|
|
command: |
|
|
# copy into a parent folder test-reports because we can't use CIRCLEI_BUILD_NUM in path when persisting to workspace
|
|
mkdir -p test-reports/test-reports_${CIRCLE_BUILD_NUM}/test/test-reports
|
|
cp -r test/test-reports test-reports/test-reports_${CIRCLE_BUILD_NUM}/test/test-reports
|
|
- store_test_results:
|
|
path: test/test-reports
|
|
- persist_to_workspace:
|
|
root: /Users/distiller/project/
|
|
paths:
|
|
- test-reports
|
|
|
|
upload_test_stats:
|
|
machine: # executor type
|
|
image: ubuntu-2004:202010-01 # # recommended linux image - includes Ubuntu 20.04, docker 19.03.13, docker-compose 1.27.4
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/workspace
|
|
- run:
|
|
name: upload
|
|
command: |
|
|
set -ex
|
|
if [ -z ${AWS_ACCESS_KEY_FOR_OSSCI_ARTIFACT_UPLOAD} ]; then
|
|
echo "No credentials found, cannot upload test stats (are you on a fork?)"
|
|
exit 0
|
|
fi
|
|
cp -r ~/workspace/test-reports/* ~/project
|
|
pip3 install requests==2.26 rockset==1.0.3 boto3==1.19.12
|
|
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_FOR_OSSCI_ARTIFACT_UPLOAD}
|
|
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_KEY_FOR_OSSCI_ARTIFACT_UPLOAD}
|
|
# i dont know how to get the run attempt number for reruns so default to 1
|
|
python3 -m tools.stats.upload_test_stats --workflow-run-id "${CIRCLE_WORKFLOW_JOB_ID}" --workflow-run-attempt 1 --head-branch << pipeline.git.branch >> --circleci
|
|
pytorch_macos_10_13_py3_test:
|
|
environment:
|
|
BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
|
|
macos:
|
|
xcode: "12.0"
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/workspace
|
|
- run_brew_for_macos_build
|
|
- run:
|
|
name: Test
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
export JOB_BASE_NAME=$CIRCLE_JOB
|
|
|
|
chmod a+x .ci/pytorch/macos-test.sh
|
|
unbuffer .ci/pytorch/macos-test.sh 2>&1 | ts
|
|
- store_test_results:
|
|
path: test/test-reports
|
|
|
|
pytorch_macos_10_13_py3_lite_interpreter_build_test:
|
|
environment:
|
|
BUILD_ENVIRONMENT: pytorch-macos-10.13-py3-test
|
|
macos:
|
|
xcode: "12.0"
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: ~/workspace
|
|
- run_brew_for_macos_build
|
|
- run:
|
|
name: Test
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
export BUILD_LITE_INTERPRETER=1
|
|
export JOB_BASE_NAME=$CIRCLE_JOB
|
|
chmod a+x ${HOME}/project/.ci/pytorch/macos-lite-interpreter-build-test.sh
|
|
unbuffer ${HOME}/project/.ci/pytorch/macos-lite-interpreter-build-test.sh 2>&1 | ts
|
|
- store_test_results:
|
|
path: test/test-reports
|
|
|
|
pytorch_android_gradle_build:
|
|
environment:
|
|
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build
|
|
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
|
|
PYTHON_VERSION: "3.7"
|
|
resource_class: large
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
steps:
|
|
- checkout
|
|
- calculate_docker_image_tag
|
|
- setup_linux_system_environment
|
|
- setup_ci_environment
|
|
- run:
|
|
name: pytorch android gradle build
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -eux
|
|
docker_image_commit=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}
|
|
|
|
docker_image_libtorch_android_x86_32=${docker_image_commit}-android-x86_32
|
|
docker_image_libtorch_android_x86_64=${docker_image_commit}-android-x86_64
|
|
docker_image_libtorch_android_arm_v7a=${docker_image_commit}-android-arm-v7a
|
|
docker_image_libtorch_android_arm_v8a=${docker_image_commit}-android-arm-v8a
|
|
|
|
echo "docker_image_commit: "${docker_image_commit}
|
|
echo "docker_image_libtorch_android_x86_32: "${docker_image_libtorch_android_x86_32}
|
|
echo "docker_image_libtorch_android_x86_64: "${docker_image_libtorch_android_x86_64}
|
|
echo "docker_image_libtorch_android_arm_v7a: "${docker_image_libtorch_android_arm_v7a}
|
|
echo "docker_image_libtorch_android_arm_v8a: "${docker_image_libtorch_android_arm_v8a}
|
|
|
|
# x86_32
|
|
time docker pull ${docker_image_libtorch_android_x86_32} >/dev/null
|
|
export id_x86_32=$(docker run --env-file "${BASH_ENV}" -e GRADLE_OFFLINE=1 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})
|
|
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
# arm-v7a
|
|
time docker pull ${docker_image_libtorch_android_arm_v7a} >/dev/null
|
|
export id_arm_v7a=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v7a})
|
|
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v7a" bash) 2>&1'
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
mkdir -p ~/workspace/build_android_install_arm_v7a
|
|
docker cp $id_arm_v7a:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_arm_v7a
|
|
|
|
# x86_64
|
|
time docker pull ${docker_image_libtorch_android_x86_64} >/dev/null
|
|
export id_x86_64=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_64})
|
|
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_x86_64" bash) 2>&1'
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
mkdir -p ~/workspace/build_android_install_x86_64
|
|
docker cp $id_x86_64:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_x86_64
|
|
|
|
# arm-v8a
|
|
time docker pull ${docker_image_libtorch_android_arm_v8a} >/dev/null
|
|
export id_arm_v8a=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_arm_v8a})
|
|
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace") | docker exec -u jenkins -i "$id_arm_v8a" bash) 2>&1'
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
mkdir -p ~/workspace/build_android_install_arm_v8a
|
|
docker cp $id_arm_v8a:/var/lib/jenkins/workspace/build_android/install ~/workspace/build_android_install_arm_v8a
|
|
|
|
docker cp ~/workspace/build_android_install_arm_v7a $id_x86_32:/var/lib/jenkins/workspace/build_android_install_arm_v7a
|
|
docker cp ~/workspace/build_android_install_x86_64 $id_x86_32:/var/lib/jenkins/workspace/build_android_install_x86_64
|
|
docker cp ~/workspace/build_android_install_arm_v8a $id_x86_32:/var/lib/jenkins/workspace/build_android_install_arm_v8a
|
|
|
|
# run gradle buildRelease
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
mkdir -p ~/workspace/build_android_artifacts
|
|
docker cp $id_x86_32:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_artifacts/
|
|
|
|
output_image=$docker_image_libtorch_android_x86_32-gradle
|
|
docker commit "$id_x86_32" ${output_image}
|
|
time docker push ${output_image}
|
|
- store_artifacts:
|
|
path: ~/workspace/build_android_artifacts/artifacts.tgz
|
|
destination: artifacts.tgz
|
|
|
|
pytorch_android_publish_snapshot:
|
|
environment:
|
|
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-publish-snapshot
|
|
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
|
|
PYTHON_VERSION: "3.7"
|
|
resource_class: large
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
steps:
|
|
- checkout
|
|
- calculate_docker_image_tag
|
|
- setup_linux_system_environment
|
|
- setup_ci_environment
|
|
- run:
|
|
name: pytorch android gradle build
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -eux
|
|
docker_image_commit=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}
|
|
|
|
docker_image_libtorch_android_x86_32_gradle=${docker_image_commit}-android-x86_32-gradle
|
|
|
|
echo "docker_image_commit: "${docker_image_commit}
|
|
echo "docker_image_libtorch_android_x86_32_gradle: "${docker_image_libtorch_android_x86_32_gradle}
|
|
|
|
# x86_32
|
|
time docker pull ${docker_image_libtorch_android_x86_32_gradle} >/dev/null
|
|
export id_x86_32=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32_gradle})
|
|
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export SONATYPE_NEXUS_USERNAME=${SONATYPE_NEXUS_USERNAME}" && echo "export SONATYPE_NEXUS_PASSWORD=${SONATYPE_NEXUS_PASSWORD}" && echo "export ANDROID_SIGN_KEY=${ANDROID_SIGN_KEY}" && echo "export ANDROID_SIGN_PASS=${ANDROID_SIGN_PASS}" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/publish_android_snapshot.sh") | docker exec -u jenkins -i "$id_x86_32" bash) 2>&1'
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
output_image=${docker_image_libtorch_android_x86_32_gradle}-publish-snapshot
|
|
docker commit "$id_x86_32" ${output_image}
|
|
time docker push ${output_image}
|
|
|
|
pytorch_android_gradle_build-x86_32:
|
|
environment:
|
|
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-only-x86_32
|
|
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c"
|
|
PYTHON_VERSION: "3.7"
|
|
resource_class: large
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
steps:
|
|
- checkout
|
|
- calculate_docker_image_tag
|
|
- setup_linux_system_environment
|
|
- checkout
|
|
- setup_ci_environment
|
|
- run:
|
|
name: pytorch android gradle build only x86_32 (for PR)
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
docker_image_libtorch_android_x86_32=${DOCKER_IMAGE}:build-${DOCKER_TAG}-${CIRCLE_SHA1}-android-x86_32
|
|
echo "docker_image_libtorch_android_x86_32: "${docker_image_libtorch_android_x86_32}
|
|
|
|
# x86
|
|
time docker pull ${docker_image_libtorch_android_x86_32} >/dev/null
|
|
export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image_libtorch_android_x86_32})
|
|
|
|
export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GRADLE_OFFLINE=1" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
mkdir -p ~/workspace/build_android_x86_32_artifacts
|
|
docker cp $id:/var/lib/jenkins/workspace/android/artifacts.tgz ~/workspace/build_android_x86_32_artifacts/
|
|
|
|
output_image=${docker_image_libtorch_android_x86_32}-gradle
|
|
docker commit "$id" ${output_image}
|
|
time docker push ${output_image}
|
|
- store_artifacts:
|
|
path: ~/workspace/build_android_x86_32_artifacts/artifacts.tgz
|
|
destination: artifacts.tgz
|
|
|
|
pytorch_ios_build:
|
|
<<: *pytorch_ios_params
|
|
macos:
|
|
xcode: "12.5.1"
|
|
steps:
|
|
- run:
|
|
name: checkout with retry
|
|
command: |
|
|
checkout() {
|
|
set -ex
|
|
# Workaround old docker images with incorrect $HOME
|
|
# check https://github.com/docker/docker/issues/2968 for details
|
|
if [ "${HOME}" = "/" ]
|
|
then
|
|
export HOME=$(getent passwd $(id -un) | cut -d: -f6)
|
|
fi
|
|
|
|
mkdir -p ~/.ssh
|
|
|
|
echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
|
|
' >> ~/.ssh/known_hosts
|
|
|
|
# use git+ssh instead of https
|
|
git config --global url."ssh://git@github.com".insteadOf "https://github.com" || true
|
|
git config --global gc.auto 0 || true
|
|
|
|
echo 'Cloning git repository'
|
|
mkdir -p '/Users/distiller/project'
|
|
cd '/Users/distiller/project'
|
|
git clone "$CIRCLE_REPOSITORY_URL" .
|
|
echo 'Checking out branch'
|
|
git checkout --force -B "$CIRCLE_BRANCH" "$CIRCLE_SHA1"
|
|
git --no-pager log --no-color -n 1 --format='HEAD is now at %h %s'
|
|
}
|
|
|
|
retry () {
|
|
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
|
}
|
|
retry checkout
|
|
- run_brew_for_ios_build
|
|
- run:
|
|
name: Setup Fastlane
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
PROJ_ROOT=/Users/distiller/project
|
|
cd ${PROJ_ROOT}/ios/TestApp
|
|
# install fastlane
|
|
sudo gem install bundler && bundle install
|
|
- run:
|
|
name: Build
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
WORKSPACE=/Users/distiller/workspace
|
|
PROJ_ROOT=/Users/distiller/project
|
|
export TCLLIBPATH="/usr/local/lib"
|
|
|
|
# Install conda
|
|
curl --retry 3 -o ~/conda.sh https://repo.anaconda.com/miniconda/Miniconda3-py39_4.12.0-MacOSX-x86_64.sh
|
|
chmod +x ~/conda.sh
|
|
/bin/bash ~/conda.sh -b -p ~/anaconda
|
|
export PATH="~/anaconda/bin:${PATH}"
|
|
source ~/anaconda/bin/activate
|
|
|
|
# Install dependencies
|
|
retry () {
|
|
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
|
}
|
|
|
|
retry conda install numpy ninja pyyaml mkl mkl-include setuptools cmake requests typing-extensions --yes
|
|
|
|
# sync submodules
|
|
cd ${PROJ_ROOT}
|
|
git submodule sync
|
|
git submodule update --init --recursive --depth 1 --jobs 0
|
|
|
|
# export
|
|
export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
|
|
|
|
# run build script
|
|
chmod a+x ${PROJ_ROOT}/scripts/build_ios.sh
|
|
echo "IOS_ARCH: ${IOS_ARCH}"
|
|
echo "IOS_PLATFORM: ${IOS_PLATFORM}"
|
|
echo "USE_PYTORCH_METAL": "${USE_METAL}"
|
|
echo "BUILD_LITE_INTERPRETER": "${BUILD_LITE_INTERPRETER}"
|
|
echo "USE_COREML_DELEGATE": "${USE_COREML_DELEGATE}"
|
|
|
|
#check the custom build flag
|
|
echo "SELECTED_OP_LIST: ${SELECTED_OP_LIST}"
|
|
if [ -n "${SELECTED_OP_LIST}" ]; then
|
|
export SELECTED_OP_LIST="${PROJ_ROOT}/ios/TestApp/custom_build/${SELECTED_OP_LIST}"
|
|
fi
|
|
export IOS_ARCH=${IOS_ARCH}
|
|
export IOS_PLATFORM=${IOS_PLATFORM}
|
|
export USE_COREML_DELEGATE=${USE_COREML_DELEGATE}
|
|
if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
|
|
export USE_PYTORCH_METAL=${USE_METAL}
|
|
fi
|
|
unbuffer ${PROJ_ROOT}/scripts/build_ios.sh 2>&1 | ts
|
|
- run:
|
|
name: Run Build Test
|
|
no_output_timeout: "30m"
|
|
command: |
|
|
set -e
|
|
PROJ_ROOT=/Users/distiller/project
|
|
# run the ruby build script
|
|
if ! [ -x "$(command -v xcodebuild)" ]; then
|
|
echo 'Error: xcodebuild is not installed.'
|
|
exit 1
|
|
fi
|
|
ruby ${PROJ_ROOT}/scripts/xcode_build.rb -i ${PROJ_ROOT}/build_ios/install -x ${PROJ_ROOT}/ios/TestApp/TestApp.xcodeproj -p ${IOS_PLATFORM}
|
|
if ! [ "$?" -eq "0" ]; then
|
|
echo 'xcodebuild failed!'
|
|
exit 1
|
|
fi
|
|
- run:
|
|
name: Run Simulator Tests
|
|
no_output_timeout: "2h"
|
|
command: |
|
|
set -e
|
|
if [ ${IOS_PLATFORM} != "SIMULATOR" ]; then
|
|
echo "not SIMULATOR build, skip it."
|
|
exit 0
|
|
fi
|
|
WORKSPACE=/Users/distiller/workspace
|
|
PROJ_ROOT=/Users/distiller/project
|
|
source ~/anaconda/bin/activate
|
|
# use the pytorch nightly build to generate models
|
|
pip3 install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
|
|
# generate models for differnet backends
|
|
cd ${PROJ_ROOT}/ios/TestApp/benchmark
|
|
mkdir -p ../models
|
|
if [ ${USE_COREML_DELEGATE} == 1 ]; then
|
|
pip install coremltools==5.0b5 protobuf==3.20.1
|
|
python coreml_backend.py
|
|
else
|
|
cd "${PROJ_ROOT}"
|
|
python test/mobile/model_test/gen_test_model.py ios-test
|
|
fi
|
|
cd "${PROJ_ROOT}/ios/TestApp/benchmark"
|
|
if [ ${BUILD_LITE_INTERPRETER} == 1 ]; then
|
|
echo "Setting up the TestApp for LiteInterpreter"
|
|
ruby setup.rb --lite 1
|
|
else
|
|
echo "Setting up the TestApp for Full JIT"
|
|
ruby setup.rb
|
|
fi
|
|
cd "${PROJ_ROOT}/ios/TestApp"
|
|
# instruments -s -devices
|
|
if [ "${BUILD_LITE_INTERPRETER}" == 1 ]; then
|
|
if [ "${USE_COREML_DELEGATE}" == 1 ]; then
|
|
fastlane scan --only_testing TestAppTests/TestAppTests/testCoreML
|
|
else
|
|
fastlane scan --only_testing TestAppTests/TestAppTests/testLiteInterpreter
|
|
fi
|
|
else
|
|
fastlane scan --only_testing TestAppTests/TestAppTests/testFullJIT
|
|
fi
|
|
pytorch_linux_bazel_build:
|
|
<<: *pytorch_params
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
steps:
|
|
- checkout
|
|
- calculate_docker_image_tag
|
|
- setup_linux_system_environment
|
|
- setup_ci_environment
|
|
- run:
|
|
name: Bazel Build
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set -e
|
|
# Pull Docker image and run build
|
|
echo "DOCKER_IMAGE: "${DOCKER_IMAGE}:${DOCKER_TAG}
|
|
time docker pull ${DOCKER_IMAGE}:${DOCKER_TAG} >/dev/null
|
|
export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE}:${DOCKER_TAG})
|
|
|
|
echo "Do NOT merge main branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
|
|
|
|
git submodule sync && git submodule update -q --init --recursive --depth 1 --jobs 0
|
|
|
|
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
|
|
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .ci/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
|
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
# Push intermediate Docker image for next phase to use
|
|
if [ -z "${BUILD_ONLY}" ]; then
|
|
# Augment our output image name with bazel to avoid collisions
|
|
output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
|
|
export COMMIT_DOCKER_IMAGE=$output_image
|
|
docker commit "$id" ${COMMIT_DOCKER_IMAGE}
|
|
time docker push ${COMMIT_DOCKER_IMAGE}
|
|
fi
|
|
|
|
pytorch_linux_bazel_test:
|
|
<<: *pytorch_params
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
steps:
|
|
- checkout
|
|
- calculate_docker_image_tag
|
|
- setup_linux_system_environment
|
|
- setup_ci_environment
|
|
- run:
|
|
name: Test
|
|
no_output_timeout: "90m"
|
|
command: |
|
|
set -e
|
|
output_image=${DOCKER_IMAGE}:build-${DOCKER_TAG}-bazel-${CIRCLE_SHA1}
|
|
export COMMIT_DOCKER_IMAGE=$output_image
|
|
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
|
|
|
|
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
|
|
|
|
if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
|
|
export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --gpus all -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
|
else
|
|
export id=$(docker run --env-file "${BASH_ENV}" --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
|
|
fi
|
|
|
|
retrieve_test_reports() {
|
|
echo "retrieving test reports"
|
|
docker cp -L $id:/var/lib/jenkins/workspace/bazel-testlogs ./ || echo 'No test reports found!'
|
|
}
|
|
trap "retrieve_test_reports" ERR
|
|
|
|
if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .ci/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
|
else
|
|
export COMMAND='((echo "sudo chown -R jenkins workspace && cd workspace && .ci/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
|
|
fi
|
|
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
|
|
|
|
retrieve_test_reports
|
|
docker stats --all --no-stream
|
|
- store_test_results:
|
|
path: bazel-testlogs
|
|
|
|
pytorch_windows_test_multigpu:
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Test
|
|
no_output_timeout: "90m"
|
|
command: |
|
|
set -e
|
|
python3 -m pip install requests
|
|
python3 ./.circleci/scripts/trigger_azure_pipeline.py
|
|
|
|
# update_s3_htmls job
|
|
# These jobs create html files for every cpu/cu## folder in s3. The html
|
|
# files just store the names of all the files in that folder (which are
|
|
# binary files (.whl files)). This is to allow pip installs of the latest
|
|
# version in a folder without having to know the latest date. Pip has a flag
|
|
# -f that you can pass an html file listing a bunch of packages, and pip will
|
|
# then install the one with the most recent version.
|
|
update_s3_htmls: &update_s3_htmls
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
resource_class: medium
|
|
steps:
|
|
- checkout
|
|
- setup_linux_system_environment
|
|
- run:
|
|
<<: *binary_checkout
|
|
# N.B. we do not run binary_populate_env. The only variable we need is
|
|
# PIP_UPLOAD_FOLDER (which is 'nightly/' for the nightlies and '' for
|
|
# releases, and sometimes other things for special cases). Instead we
|
|
# expect PIP_UPLOAD_FOLDER to be passed directly in the env. This is
|
|
# because, unlike all the other binary jobs, these jobs only get run once,
|
|
# in a separate workflow. They are not a step in other binary jobs like
|
|
# build, test, upload.
|
|
#
|
|
# You could attach this to every job, or include it in the upload step if
|
|
# you wanted. You would need to add binary_populate_env in this case to
|
|
# make sure it has the same upload folder as the job it's attached to. This
|
|
# function is idempotent, so it won't hurt anything; it's just a little
|
|
# unnescessary"
|
|
- run:
|
|
name: define PIP_UPLOAD_FOLDER
|
|
command: |
|
|
our_upload_folder=nightly/
|
|
# On tags upload to test instead
|
|
if [[ -n "${CIRCLE_TAG}" ]]; then
|
|
our_upload_folder=test/
|
|
fi
|
|
echo "export PIP_UPLOAD_FOLDER=${our_upload_folder}" >> ${BASH_ENV}
|
|
- run:
|
|
name: Update s3 htmls
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set +x
|
|
echo "declare -x \"AWS_ACCESS_KEY_ID=${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}\"" >> /home/circleci/project/env
|
|
echo "declare -x \"AWS_SECRET_ACCESS_KEY=${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}\"" >> /home/circleci/project/env
|
|
source /home/circleci/project/env
|
|
set -eux -o pipefail
|
|
retry () {
|
|
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
|
}
|
|
retry pip install awscli==1.6
|
|
"/home/circleci/project/builder/cron/update_s3_htmls.sh"
|
|
|
|
# There is currently no testing for libtorch TODO
|
|
# binary_linux_libtorch_3.6m_cpu_test:
|
|
# environment:
|
|
# BUILD_ENVIRONMENT: "libtorch 3.6m cpu"
|
|
# resource_class: gpu.nvidia.small
|
|
# <<: *binary_linux_test
|
|
#
|
|
# binary_linux_libtorch_3.6m_cu90_test:
|
|
# environment:
|
|
# BUILD_ENVIRONMENT: "libtorch 3.6m cu90"
|
|
# resource_class: gpu.nvidia.small
|
|
# <<: *binary_linux_test
|
|
#
|
|
docker_build_job:
|
|
parameters:
|
|
image_name:
|
|
type: string
|
|
default: ""
|
|
machine:
|
|
image: ubuntu-2004:202104-01
|
|
resource_class: large
|
|
environment:
|
|
IMAGE_NAME: << parameters.image_name >>
|
|
# Enable 'docker manifest'
|
|
DOCKER_CLI_EXPERIMENTAL: "enabled"
|
|
DOCKER_BUILDKIT: 1
|
|
steps:
|
|
- checkout
|
|
- calculate_docker_image_tag
|
|
- run:
|
|
name: Check if image should be built
|
|
command: |
|
|
set +x
|
|
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
|
|
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
|
|
export AWS_ACCOUNT_ID=$(aws sts get-caller-identity|grep Account|cut -f4 -d\")
|
|
export AWS_REGION=us-east-1
|
|
aws ecr get-login-password --region $AWS_REGION|docker login --username AWS \
|
|
--password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com
|
|
set -x
|
|
# Check if image already exists, if it does then skip building it
|
|
if docker manifest inspect "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${IMAGE_NAME}:${DOCKER_TAG}"; then
|
|
circleci-agent step halt
|
|
# circleci-agent step halt doesn't actually halt the step so we need to
|
|
# explicitly exit the step here ourselves before it causes too much trouble
|
|
exit 0
|
|
fi
|
|
# Covers the case where a previous tag doesn't exist for the tree
|
|
# this is only really applicable on trees that don't have `.ci/docker` at its merge base, i.e. nightly
|
|
if ! git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):.ci/docker"; then
|
|
echo "Directory '.ci/docker' not found in tree << pipeline.git.base_revision >>, you should probably rebase onto a more recent commit"
|
|
exit 1
|
|
fi
|
|
PREVIOUS_DOCKER_TAG=$(git rev-parse "$(git merge-base HEAD << pipeline.git.base_revision >>):ci/docker")
|
|
# If no image exists but the hash is the same as the previous hash then we should error out here
|
|
if [[ "${PREVIOUS_DOCKER_TAG}" = "${DOCKER_TAG}" ]]; then
|
|
echo "ERROR: Something has gone wrong and the previous image isn't available for the merge-base of your branch"
|
|
echo " contact the PyTorch team to restore the original images"
|
|
exit 1
|
|
fi
|
|
- run:
|
|
name: build_docker_image_<< parameters.image_name >>
|
|
no_output_timeout: "1h"
|
|
command: |
|
|
set +x
|
|
export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_DOCKER_BUILDER_V1}
|
|
export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_DOCKER_BUILDER_V1}
|
|
set -x
|
|
cd .ci/docker && ./build_docker.sh
|
|
##############################################################################
|
|
# Workflows
|
|
##############################################################################
|
|
workflows:
|
|
build:
|
|
jobs:
|
|
- binary_ios_build:
|
|
build_environment: libtorch-ios-12.5.1-nightly-x86_64-build
|
|
context: org-member
|
|
filters:
|
|
branches:
|
|
only: nightly
|
|
ios_arch: x86_64
|
|
ios_platform: SIMULATOR
|
|
name: pytorch_ios_12_5_1_nightly_x86_64_build
|
|
use_coreml: "1"
|
|
use_metal: "1"
|
|
- binary_ios_build:
|
|
build_environment: libtorch-ios-12.5.1-nightly-arm64-build
|
|
context: org-member
|
|
filters:
|
|
branches:
|
|
only: nightly
|
|
ios_arch: arm64
|
|
ios_platform: OS
|
|
name: pytorch_ios_12_5_1_nightly_arm64_build
|
|
use_coreml: "1"
|
|
use_metal: "1"
|
|
- binary_ios_build:
|
|
build_environment: libtorch-ios-full_jit-12.5.1-nightly-x86_64-build
|
|
context: org-member
|
|
filters:
|
|
branches:
|
|
only: nightly
|
|
ios_arch: x86_64
|
|
ios_platform: SIMULATOR
|
|
lite_interpreter: "0"
|
|
name: pytorch_ios_full_jit_12_5_1_nightly_x86_64_build
|
|
use_coreml: "1"
|
|
use_metal: "1"
|
|
- binary_ios_build:
|
|
build_environment: libtorch-ios-full_jit-12.5.1-nightly-arm64-build
|
|
context: org-member
|
|
filters:
|
|
branches:
|
|
only: nightly
|
|
ios_arch: arm64
|
|
ios_platform: OS
|
|
lite_interpreter: "0"
|
|
name: pytorch_ios_full_jit_12_5_1_nightly_arm64_build
|
|
use_coreml: "1"
|
|
use_metal: "1"
|
|
- binary_ios_upload:
|
|
build_environment: libtorch-ios-12.5.1-nightly-binary-build-upload
|
|
context: org-member
|
|
filters:
|
|
branches:
|
|
only: nightly
|
|
requires:
|
|
- pytorch_ios_12_5_1_nightly_x86_64_build
|
|
- pytorch_ios_12_5_1_nightly_arm64_build
|
|
- binary_ios_upload:
|
|
build_environment: libtorch-ios-full_jit-12.5.1-nightly-binary-build-upload
|
|
context: org-member
|
|
filters:
|
|
branches:
|
|
only: nightly
|
|
lite_interpreter: "0"
|
|
requires:
|
|
- pytorch_ios_full_jit_12_5_1_nightly_x86_64_build
|
|
- pytorch_ios_full_jit_12_5_1_nightly_arm64_build
|
|
- anaconda_prune:
|
|
name: anaconda-prune-pytorch-nightly
|
|
context: "org-member"
|
|
packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng"
|
|
channel: pytorch-nightly
|
|
filters:
|
|
branches:
|
|
only:
|
|
- postnightly
|
|
- anaconda_prune:
|
|
name: anaconda-prune-pytorch-test
|
|
context: "org-member"
|
|
packages: "pytorch torchvision torchaudio torchtext ignite torchcsprng"
|
|
channel: pytorch-test
|
|
filters:
|
|
branches:
|
|
only:
|
|
- postnightly
|
|
when: << pipeline.parameters.run_build >>
|