Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update expat for Pytorch training 1.13 #4267

Closed
wants to merge 7 commits into from
Closed

Update expat for Pytorch training 1.13 #4267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
80a4c73
Update ['dlc_developer_config.toml']
kc1998dp Sep 19, 2024
438d1e7
Update expat for Pytorch training 1.13
kc1998dp Sep 19, 2024
6bcd3ee
Update gevent and pyarrow for Pytorch training 1.13
kc1998dp Sep 19, 2024
7c7c3da
Revert "Update gevent and pyarrow for Pytorch training 1.13"
kc1998dp Sep 19, 2024
cf437f4
Ignore unaddressable Linux kernel vulnerability for pytorch training …
kc1998dp Sep 19, 2024
c180398
ignore cve-2023-7018
kc1998dp Sep 19, 2024
f906046
Merge branch 'master' into master
kc1998dp Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 7 additions & 92 deletions dlc_developer_config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,159 +1,74 @@
[dev]
# Set to "huggingface", for example, if you are a huggingface developer. Default is ""
partner_developer = ""
# Please only set it to true if you are preparing an EI related PR
# Do remember to revert it back to false before merging any PR (including EI dedicated PR)
ei_mode = false
# Please only set it to true if you are preparing a NEURON related PR
# Do remember to revert it back to false before merging any PR (including NEURON dedicated PR)
neuron_mode = false
# Please only set it to true if you are preparing a NEURONX related PR
# Do remember to revert it back to false before merging any PR (including NEURONX dedicated PR)
neuronx_mode = false
# Please only set it to true if you are preparing a GRAVITON related PR
# Do remember to revert it back to false before merging any PR (including GRAVITON dedicated PR)
graviton_mode = false
# Please only set it to True if you are preparing a HABANA related PR
# Do remember to revert it back to False before merging any PR (including HABANA dedicated PR)
habana_mode = false
# Please only set it to True if you are preparing a HUGGINGFACE TRCOMP related PR
# Do remember to revert it back to False before merging any PR (including HUGGINGFACE TRCOMP dedicated PR)
# This mode is used to build TF 2.6 and PT1.11 DLC
huggingface_trcomp_mode = false
# Please only set it to True if you are preparing a TRCOMP related PR
# Do remember to revert it back to False before merging any PR (including TRCOMP dedicated PR)
# This mode is used to build PT1.12 and above DLC
trcomp_mode = false
# Set deep_canary_mode to true to simulate Deep Canary Test conditions on PR for all frameworks in the
# build_frameworks list below. This will cause all image builds and non-deep-canary tests on the PR to be skipped,
# regardless of whether they are enabled or disabled below.
# Set graviton_mode to true to run Deep Canaries on Graviton images.
# Do remember to revert it back to false before merging any PR.
deep_canary_mode = false

[build]
# Add in frameworks you would like to build. By default, builds are disabled unless you specify building an image.
# available frameworks - ["autogluon", "huggingface_tensorflow", "huggingface_pytorch", "huggingface_tensorflow_trcomp", "huggingface_pytorch_trcomp", "pytorch_trcomp", "tensorflow", "mxnet", "pytorch", "stabilityai_pytorch"]
build_frameworks = []

# By default we build both training and inference containers. Set true/false values to determine which to build.
build_frameworks = [ "huggingface_pytorch",]
build_training = true
build_inference = true

# Set do_build to "false" to skip builds and test the latest image built by this PR
# Note: at least one build is required to set do_build to "false"
build_inference = false
do_build = true

[notify]
### Notify on test failures
### Off by default
notify_test_failures = false
# Valid values: medium or high
notification_severity = "medium"
notification_severity = "medium"

[test]
### On by default
sanity_tests = true
safety_check_test = false
ecr_scan_allowlist_feature = false
safety_check_test = false
ecr_scan_allowlist_feature = false
ecs_tests = true
eks_tests = true
ec2_tests = true
# Set it to true if you are preparing a Benchmark related PR
ec2_benchmark_tests = false

### Set ec2_tests_on_heavy_instances = true to be able to run any EC2 tests that use large/expensive instance types by
### default. If false, these types of tests will be skipped while other tests will run as usual.
### These tests are run in EC2 test jobs, so ec2_tests must be true if ec2_tests_on_heavy_instances is true.
### Off by default (set to false)
ec2_tests_on_heavy_instances = false

### SM specific tests
### On by default
sagemaker_local_tests = true

# run standard sagemaker remote tests from test/sagemaker_tests
sagemaker_remote_tests = true
# run efa sagemaker tests
sagemaker_efa_tests = false
# run release_candidate_integration tests
sagemaker_rc_tests = false
# run sagemaker benchmark tests
sagemaker_benchmark_tests = false

# SM remote EFA test instance type
sagemaker_remote_efa_instance_type = ""

# Run CI tests for nightly images
# false by default
nightly_pr_test_mode = false

use_scheduler = false

[buildspec_override]
# Assign the path to the required buildspec file from the deep-learning-containers folder
# For example:
# dlc-pr-tensorflow-2-habana-training = "habana/tensorflow/training/buildspec-2-10.yml"
# dlc-pr-pytorch-inference = "pytorch/inference/buildspec-1-12.yml"
# Setting the buildspec file path to "" allows the image builder to choose the default buildspec file.

### TRAINING PR JOBS ###

# Standard Framework Training
dlc-pr-mxnet-training = ""
dlc-pr-pytorch-training = ""
dlc-pr-tensorflow-2-training = ""
dlc-pr-autogluon-training = ""

# HuggingFace Training
dlc-pr-huggingface-tensorflow-training = ""
dlc-pr-huggingface-pytorch-training = ""

# Training Compiler
dlc-pr-huggingface-pytorch-training = "huggingface/pytorch/training/buildspec-1-13.yml"
dlc-pr-huggingface-pytorch-trcomp-training = ""
dlc-pr-huggingface-tensorflow-2-trcomp-training = ""
dlc-pr-pytorch-trcomp-training = ""

# Neuron Training
dlc-pr-mxnet-neuron-training = ""
dlc-pr-pytorch-neuron-training = ""
dlc-pr-tensorflow-2-neuron-training = ""

# Stability AI Training
dlc-pr-stabilityai-pytorch-training = ""

# Habana Training
dlc-pr-pytorch-habana-training = ""
dlc-pr-tensorflow-2-habana-training = ""

### INFERENCE PR JOBS ###

# Standard Framework Inference
dlc-pr-mxnet-inference = ""
dlc-pr-pytorch-inference = ""
dlc-pr-tensorflow-2-inference = ""
dlc-pr-autogluon-inference = ""

# Neuron Inference
dlc-pr-mxnet-neuron-inference = ""
dlc-pr-pytorch-neuron-inference = ""
dlc-pr-tensorflow-1-neuron-inference = ""
dlc-pr-tensorflow-2-neuron-inference = ""

# HuggingFace Inference
dlc-pr-huggingface-tensorflow-inference = ""
dlc-pr-huggingface-pytorch-inference = ""
dlc-pr-huggingface-pytorch-neuron-inference = ""

# Stability AI Inference
dlc-pr-stabilityai-pytorch-inference = ""

# Graviton Inference
dlc-pr-mxnet-graviton-inference = ""
dlc-pr-pytorch-graviton-inference = ""
dlc-pr-tensorflow-2-graviton-inference = ""

# EIA Inference
dlc-pr-mxnet-eia-inference = ""
dlc-pr-pytorch-eia-inference = ""
dlc-pr-tensorflow-2-eia-inference = ""

2 changes: 1 addition & 1 deletion huggingface/pytorch/training/docker/1.13/py3/cu117/Dockerfile.gpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ RUN pip install --no-cache-dir \
RUN apt-get update \
# TODO: Remove upgrade statements once packages are updated in base image
&& apt-get -y upgrade --only-upgrade systemd openssl cryptsetup \
&& apt install -y git git-lfs \
&& apt install -y git git-lfs expat \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

Expand Down
Loading
Loading