Replies: 2 comments
-
|
Hi @jmarkow, We'll dig into why we're not calculating the padding correctly, but can you try adjusting these lines in your config and seeing if you still have the same issue? Old: New: Thanks! Talmo |
Beta Was this translation helpful? Give feedback.
0 replies
-
|
Such fast troubleshooting! Works now Also hey from sunny Atlanta @talmo . |
Beta Was this translation helpful? Give feedback.
0 replies
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
-
Sleap version 1.3.3. Running
sleap-trainCLI in an HPC environment (CentOS). Videos are 8 bit grayscale FFV1-encoded AVI files. Training with the Unet backbone works fine, however the pretrained encoder appears to have rounding errors when creating the backbone. Looks similar to issue #826 , but I'm running with the "auto-crop" option enabled AFAIK.Here's the training json file.
{ "data": { "labels": { "training_labels": "../_labels/merged_labels_Basler-267601741000-24383488.slp", "validation_labels": null, "validation_fraction": 0.2, "test_labels": null, "split_by_inds": false, "training_inds": null, "validation_inds": null, "test_inds": null, "search_path_hints": [ "../../training_data/Basler-267601746223-24404515", "../../training_data/Basler-267601746219-24404505", "../../training_data/Basler-2676017611A4-24514980", "../../training_data/Basler-26760175421F-24461855", "../../training_data/Basler-267601741000-24383488" ], "skeletons": [] }, "preprocessing": { "ensure_rgb": false, "ensure_grayscale": false, "imagenet_mode": null, "input_scaling": 1.0, "pad_to_stride": 16, "resize_and_pad_to_target": true, "target_height": 1023, "target_width": 1008 }, "instance_cropping": { "center_on_part": null, "crop_size": null, "crop_size_detection_padding": 16 } }, "model": { "backbone": { "leap": null, "unet": null, "hourglass": null, "resnet": null, "pretrained_encoder": { "encoder": "densenet121", "pretrained": true, "decoder_filters": 256, "decoder_filters_rate": 1.0, "output_stride": 2, "decoder_batchnorm": true } }, "heads": { "single_instance": { "part_names": [ "nose", "ear_L", "ear_R", "foreleg_L", "foreleg_R", "hindleg_L", "hindleg_R", "back_top", "back_middle", "back_bottom", "tail_base", "tail_middle", "tail_tip" ], "sigma": 2.5, "output_stride": 4, "loss_weight": 1.0, "offset_refinement": false }, "centroid": null, "centered_instance": null, "multi_instance": null, "multi_class_bottomup": null, "multi_class_topdown": null }, "base_checkpoint": null }, "optimization": { "preload_data": true, "augmentation_config": { "rotate": true, "rotation_min_angle": -30, "rotation_max_angle": 30, "translate": true, "translate_min": -100, "translate_max": 100, "scale": true, "scale_min": 0.7, "scale_max": 1.3, "uniform_noise": false, "uniform_noise_min_val": 0.0, "uniform_noise_max_val": 10.0, "gaussian_noise": true, "gaussian_noise_mean": 5.0, "gaussian_noise_stddev": 5.0, "contrast": true, "contrast_min_gamma": 0.5, "contrast_max_gamma": 2.0, "brightness": true, "brightness_min_val": -30, "brightness_max_val": 30, "random_crop": false, "random_crop_height": 256, "random_crop_width": 256, "random_flip": false, "flip_horizontal": false }, "online_shuffling": true, "shuffle_buffer_size": 128, "prefetch": true, "batch_size": 4, "batches_per_epoch": 200, "min_batches_per_epoch": 400, "val_batches_per_epoch": 10, "min_val_batches_per_epoch": 10, "epochs": 100, "optimizer": "adam", "initial_learning_rate": 0.0001, "learning_rate_schedule": { "reduce_on_plateau": true, "reduction_factor": 0.5, "plateau_min_delta": 1e-06, "plateau_patience": 5, "plateau_cooldown": 3, "min_learning_rate": 1e-08 }, "hard_keypoint_mining": { "online_mining": false, "hard_to_easy_ratio": 2.0, "min_hard_keypoints": 2, "max_hard_keypoints": null, "loss_scale": 5.0 }, "early_stopping": { "stop_training_on_plateau": true, "plateau_min_delta": 1e-08, "plateau_patience": 10 } }, "outputs": { "save_outputs": true, "run_name": "unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0", "run_name_prefix": "", "run_name_suffix": ".single_instance", "runs_folder": "/Users/jeffreymarkowitz/Desktop/basler_plexiglass_sleap_training/labeling_project/models", "tags": [ "" ], "save_visualizations": true, "delete_viz_images": true, "zip_outputs": false, "log_to_csv": true, "checkpointing": { "initial_model": false, "best_model": true, "every_epoch": false, "latest_model": false, "final_model": false }, "tensorboard": { "write_logs": false, "loss_frequency": "epoch", "architecture_graph": false, "profile_graph": false, "visualizations": true }, "zmq": { "subscribe_to_controller": true, "controller_address": "tcp://127.0.0.1:9000", "controller_polling_timeout": 10, "publish_updates": true, "publish_address": "tcp://127.0.0.1:9001" } }, "name": "", "description": "", "sleap_version": "1.3.3", "filename": "single_instance.json" }and log from run,
--------------------------------------- Begin Slurm Prolog: Apr-03-2024 11:36:26 Job ID: 5503480 User ID: jmarkowitz30 Account: gts-jmarkowitz30-paid Job name: wrap Partition: gpu-rtx6000 QOS: inferno --------------------------------------- INFO:sleap.nn.training:Versions: SLEAP: 1.3.3 TensorFlow: 2.7.0 Numpy: 1.19.5 Python: 3.7.12 OS: Linux-3.10.0-1160.49.1.el7.x86_64-x86_64-with-redhat-7.9-Maipo INFO:sleap.nn.training:Training labels file: None INFO:sleap.nn.training:Training profile: /storage/coda1/p-jmarkowitz30/0/shared/active_lab_members/markowitz_jeffrey/active_projects/keypoints_basler_nir_plexiglass_arena/sleap_training/gridsearch_pretrained_1/unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0.json INFO:sleap.nn.training: INFO:sleap.nn.training:Arguments: INFO:sleap.nn.training:{ "training_job_path": "/storage/coda1/p-jmarkowitz30/0/shared/active_lab_members/markowitz_jeffrey/active_projects/keypoints_basler_nir_plexiglass_arena/sleap_training/gridsearch_pretrained_1/unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0.json", "labels_path": null, "video_paths": [ "" ], "val_labels": null, "test_labels": null, "base_checkpoint": null, "tensorboard": false, "save_viz": false, "zmq": false, "run_name": "", "prefix": "", "suffix": "", "cpu": false, "first_gpu": false, "last_gpu": false, "gpu": "auto" } INFO:sleap.nn.training: INFO:sleap.nn.training:Training job: INFO:sleap.nn.training:{ "data": { "labels": { "training_labels": "../_labels/merged_labels_Basler-267601741000-24383488.slp", "validation_labels": null, "validation_fraction": 0.2, "test_labels": null, "split_by_inds": false, "training_inds": null, "validation_inds": null, "test_inds": null, "search_path_hints": [ "../../training_data/Basler-267601746223-24404515", "../../training_data/Basler-267601746219-24404505", "../../training_data/Basler-2676017611A4-24514980", "../../training_data/Basler-26760175421F-24461855", "../../training_data/Basler-267601741000-24383488" ], "skeletons": [] }, "preprocessing": { "ensure_rgb": false, "ensure_grayscale": false, "imagenet_mode": null, "input_scaling": 1.0, "pad_to_stride": 16, "resize_and_pad_to_target": true, "target_height": 1023, "target_width": 1008 }, "instance_cropping": { "center_on_part": null, "crop_size": null, "crop_size_detection_padding": 16 } }, "model": { "backbone": { "leap": null, "unet": null, "hourglass": null, "resnet": null, "pretrained_encoder": { "encoder": "densenet121", "pretrained": true, "decoder_filters": 256, "decoder_filters_rate": 1.0, "output_stride": 2, "decoder_batchnorm": true } }, "heads": { "single_instance": { "part_names": [ "nose", "ear_L", "ear_R", "foreleg_L", "foreleg_R", "hindleg_L", "hindleg_R", "back_top", "back_middle", "back_bottom", "tail_base", "tail_middle", "tail_tip" ], "sigma": 2.5, "output_stride": 4, "loss_weight": 1.0, "offset_refinement": false }, "centroid": null, "centered_instance": null, "multi_instance": null, "multi_class_bottomup": null, "multi_class_topdown": null }, "base_checkpoint": null }, "optimization": { "preload_data": true, "augmentation_config": { "rotate": true, "rotation_min_angle": -30.0, "rotation_max_angle": 30.0, "translate": true, "translate_min": -100, "translate_max": 100, "scale": true, "scale_min": 0.7, "scale_max": 1.3, "uniform_noise": false, "uniform_noise_min_val": 0.0, "uniform_noise_max_val": 10.0, "gaussian_noise": true, "gaussian_noise_mean": 5.0, "gaussian_noise_stddev": 5.0, "contrast": true, "contrast_min_gamma": 0.5, "contrast_max_gamma": 2.0, "brightness": true, "brightness_min_val": -30.0, "brightness_max_val": 30.0, "random_crop": false, "random_crop_height": 256, "random_crop_width": 256, "random_flip": false, "flip_horizontal": false }, "online_shuffling": true, "shuffle_buffer_size": 128, "prefetch": true, "batch_size": 4, "batches_per_epoch": 200, "min_batches_per_epoch": 400, "val_batches_per_epoch": 10, "min_val_batches_per_epoch": 10, "epochs": 100, "optimizer": "adam", "initial_learning_rate": 0.0001, "learning_rate_schedule": { "reduce_on_plateau": true, "reduction_factor": 0.5, "plateau_min_delta": 1e-06, "plateau_patience": 5, "plateau_cooldown": 3, "min_learning_rate": 1e-08 }, "hard_keypoint_mining": { "online_mining": false, "hard_to_easy_ratio": 2.0, "min_hard_keypoints": 2, "max_hard_keypoints": null, "loss_scale": 5.0 }, "early_stopping": { "stop_training_on_plateau": true, "plateau_min_delta": 1e-08, "plateau_patience": 10 } }, "outputs": { "save_outputs": true, "run_name": "unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0", "run_name_prefix": "", "run_name_suffix": ".single_instance", "runs_folder": "/Users/jeffreymarkowitz/Desktop/basler_plexiglass_sleap_training/labeling_project/models", "tags": [ "" ], "save_visualizations": true, "delete_viz_images": true, "zip_outputs": false, "log_to_csv": true, "checkpointing": { "initial_model": false, "best_model": true, "every_epoch": false, "latest_model": false, "final_model": false }, "tensorboard": { "write_logs": false, "loss_frequency": "epoch", "architecture_graph": false, "profile_graph": false, "visualizations": true }, "zmq": { "subscribe_to_controller": true, "controller_address": "tcp://127.0.0.1:9000", "controller_polling_timeout": 10, "publish_updates": true, "publish_address": "tcp://127.0.0.1:9001" } }, "name": "", "description": "", "sleap_version": "1.3.3", "filename": "/storage/coda1/p-jmarkowitz30/0/shared/active_lab_members/markowitz_jeffrey/active_projects/keypoints_basler_nir_plexiglass_arena/sleap_training/gridsearch_pretrained_1/unet_decoder_filters-256_decoder_filters_rate-1.0_encoder-densenet121_pretrained-True_repeat-0.json" } INFO:sleap.nn.training: INFO:sleap.nn.training:Auto-selected GPU 0 with 24208 MiB of free memory. INFO:sleap.nn.training:Using GPU 0 for acceleration. INFO:sleap.nn.training:Disabled GPU memory pre-allocation. INFO:sleap.nn.training:System: GPUs: 1/1 available Device: /physical_device:GPU:0 Available: True Initalized: False Memory growth: True INFO:sleap.nn.training: INFO:sleap.nn.training:Initializing trainer... INFO:sleap.nn.training:Loading training labels from: ../_labels/merged_labels_Basler-267601741000-24383488.slp INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.2 INFO:sleap.nn.training: Splits: Training = 331 / Validation = 83. INFO:sleap.nn.training:Setting up for training... INFO:sleap.nn.training:Setting up pipeline builders... INFO:sleap.nn.training:Setting up model... INFO:sleap.nn.training:Building test pipeline... 2024-04-03 11:36:47.108576: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2024-04-03 11:36:47.978719: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 22842 MB memory: -> device: 0, name: Quadro RTX 6000, pci bus id: 0000:af:00.0, compute capability: 7.5 INFO:sleap.nn.training:Loaded test example. [4.204s] INFO:sleap.nn.training: Input shape: (1024, 1008, 1) Downloading data from https://github.com/keras-team/keras-applications/releases/download/densenet/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5 16384/29084464 [..............................] - ETA: 1s������������������������������������������������������������ 491520/29084464 [..............................] - ETA: 2s������������������������������������������������������������ 5062656/29084464 [====>.........................] - ETA: 0s������������������������������������������������������������ 22724608/29084464 [======================>.......] - ETA: 0s������������������������������������������������������������ 29089792/29084464 [==============================] - 0s 0us/step ���������������������������������������������������������������� 29097984/29084464 [==============================] - 0s 0us/step Traceback (most recent call last): File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/bin/sleap-train", line 33, in <module> sys.exit(load_entry_point('sleap==1.3.3', 'console_scripts', 'sleap-train')()) File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 2014, in main trainer.train() File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 924, in train self.setup() File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 910, in setup self._setup_model() File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/training.py", line 734, in _setup_model self.model.make_model(input_shape) File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/model.py", line 326, in make_model x_main, x_mid = self.backbone.make_backbone(x_in=x_in) File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/sleap/nn/architectures/pretrained_encoders.py", line 226, in make_backbone utils=tf.keras.utils, File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/segmentation_models/models/unet.py", line 240, in Unet use_batchnorm=decoder_use_batchnorm, File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/segmentation_models/models/unet.py", line 139, in build_unet x = decoder_block(decoder_filters[i], stage=i, use_batchnorm=use_batchnorm)(x, skip) File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/segmentation_models/models/unet.py", line 60, in wrapper x = layers.Concatenate(axis=concat_axis, name=concat_name)([x, skip]) File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler raise e.with_traceback(filtered_tb) from None File "/storage/home/hcoda1/4/jmarkowitz30/p-jmarkowitz30-0/dev/miniconda3/envs/sleap/lib/python3.7/site-packages/keras/layers/merge.py", line 525, in build raise ValueError(err_msg) ValueError: A `Concatenate` layer requires inputs with matching shapes except for the concatenation axis. Received: input_shape=[(None, 64, 62, 1024), (None, 64, 63, 512)] --------------------------------------- Begin Slurm Epilog: Apr-03-2024 11:36:55 Job ID: 5503480 Array Job ID: _4294967294 User ID: jmarkowitz30 Account: gts-jmarkowitz30-paid Job name: wrap Resources: cpu=12,gres/gpu:rtx_6000=1,mem=100G,node=1 Rsrc Used: cput=00:06:00,vmem=800008K,walltime=00:00:30,mem=0,energy_used=0 Partition: gpu-rtx6000 QOS: inferno Nodes: atl1-1-03-006-27-0 ---------------------------------------If needed output of
pip freezeBeta Was this translation helpful? Give feedback.
All reactions