From e6e3047103d6e2fb51af575535889c79e77729fb Mon Sep 17 00:00:00 2001 From: lmeribal Date: Wed, 11 Sep 2024 17:05:29 +0300 Subject: [PATCH] in progress --- .../pipelines/preprocessing/multimodal.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/turbo_alignment/pipelines/preprocessing/multimodal.py b/turbo_alignment/pipelines/preprocessing/multimodal.py index 39d1845..81b7156 100644 --- a/turbo_alignment/pipelines/preprocessing/multimodal.py +++ b/turbo_alignment/pipelines/preprocessing/multimodal.py @@ -100,12 +100,13 @@ def _async_process_files(self, reader, encoder, files_paths, experiment_settings try: logger.info(f'📖 Encoding batch {i} / {len(batches)}') batch_output = self._process_function(reader, encoder, batch, experiment_settings, i) - logger.info(f'📖 Saving batch {i}...') - with ThreadPoolExecutor() as executor: - futures = [executor.submit(self._save_tensor, tensor, path, experiment_settings) - for path, tensor in batch_output.items()] - for future in futures: - future.result() + torch.save(batch_output, experiment_settings.output_file_path / ( + 'process_' + str(self.accelerator.process_index) + 'batch_' + str(i) + '.' + experiment_settings.modality.value + + '.' + + experiment_settings.encoder_settings.modality_encoder_type + + '.pt' + ) + ) except Exception as exc: logger.error(f'Error reading file: {exc}')