Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MNT, DOC] Accelerating deep testing #1904

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
14 changes: 10 additions & 4 deletions aeon/classification/deep_learning/_cnn.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,12 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"avg_pool_size": 4,
"n_layers": 1,
"n_filters": 1,
"kernel_size": 2,
"avg_pool_size": 2,
}

test_params = [param1]
Expand Down Expand Up @@ -652,9 +655,12 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"avg_pool_size": 4,
"n_layers": 1,
"n_filters": 1,
"kernel_size": 2,
"avg_pool_size": 2,
}

test_params = [param1]
Expand Down
2 changes: 2 additions & 0 deletions aeon/classification/deep_learning/_encoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,8 @@ def get_test_params(cls, parameter_set="default"):
"n_epochs": 8,
"batch_size": 4,
"use_bias": False,
"n_filters": [2],
"kernel_size": [2],
"fc_units": 8,
"strides": 2,
"dropout_proba": 0,
Expand Down
2 changes: 1 addition & 1 deletion aeon/classification/deep_learning/_fcn.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"use_bias": False,
"n_layers": 1,
Expand Down
21 changes: 15 additions & 6 deletions aeon/classification/deep_learning/_inception_time.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,11 +364,16 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_classifiers": 1,
"n_epochs": 10,
"n_classifiers": 2,
"n_epochs": 2,
"batch_size": 4,
"kernel_size": 4,
"depth": 1,
"kernel_size": 2,
"n_filters": 1,
"n_conv_per_layer": 1,
"use_residual": False,
"use_bottleneck": False,
"use_max_pooling": False,
"depth": 1,
"use_custom_filters": False,
}
Expand Down Expand Up @@ -747,11 +752,15 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"kernel_size": 4,
"depth": 1,
"kernel_size": 2,
"n_filters": 1,
"n_conv_per_layer": 1,
"use_residual": False,
"use_bottleneck": True,
"use_bottleneck": False,
"use_max_pooling": False,
"depth": 1,
"use_custom_filters": False,
}
Expand Down
2 changes: 1 addition & 1 deletion aeon/classification/deep_learning/_mlp.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param1 = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"use_bias": False,
}
Expand Down
2 changes: 1 addition & 1 deletion aeon/classification/deep_learning/_resnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ def get_test_params(cls, parameter_set="default"):
`create_test_instance` uses the first (or only) dictionary in `params`.
"""
param = {
"n_epochs": 10,
"n_epochs": 2,
"batch_size": 4,
"n_residual_blocks": 1,
"n_filters": 5,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,15 @@ def test_random_state_deep_learning_cls(deep_cls):

X, y = make_example_3d_numpy(random_state=random_state)

deep_cls1 = deep_cls(random_state=random_state, n_epochs=4)
test_params = deep_cls.get_test_params()[0]
test_params["random_state"] = random_state

deep_cls1 = deep_cls(**test_params)
deep_cls1.fit(X, y)

layers1 = deep_cls1.training_model_.layers[1:]

deep_cls2 = deep_cls(random_state=random_state, n_epochs=4)
deep_cls2 = deep_cls(**test_params)
deep_cls2.fit(X, y)

layers2 = deep_cls2.training_model_.layers[1:]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,35 +46,35 @@ def test_saving_loading_deep_learning_cls(deep_cls):

X, y = make_example_3d_numpy()

deep_cls_train = deep_cls(
n_epochs=2,
save_best_model=True,
save_last_model=True,
save_init_model=True,
best_file_name=best_file_name,
last_file_name=last_file_name,
init_file_name=init_file_name,
file_path=tmp,
)
test_params = deep_cls.get_test_params()[0]
test_params["save_best_model"] = True
test_params["save_last_model"] = True
test_params["save_init_model"] = True
test_params["best_file_name"] = best_file_name
test_params["last_file_name"] = last_file_name
test_params["init_file_name"] = init_file_name
test_params["file_path"] = tmp

deep_cls_train = deep_cls(**test_params)
deep_cls_train.fit(X, y)

deep_cls_best = deep_cls()
deep_cls_best = deep_cls(**test_params)
deep_cls_best.load_model(
model_path=os.path.join(tmp, best_file_name + ".keras"),
classes=np.unique(y),
)
ypred_best = deep_cls_best.predict(X)
assert len(ypred_best) == len(y)

deep_cls_last = deep_cls()
deep_cls_last = deep_cls(**test_params)
deep_cls_last.load_model(
model_path=os.path.join(tmp, last_file_name + ".keras"),
classes=np.unique(y),
)
ypred_last = deep_cls_last.predict(X)
assert len(ypred_last) == len(y)

deep_cls_init = deep_cls()
deep_cls_init = deep_cls(**test_params)
deep_cls_init.load_model(
model_path=os.path.join(tmp, init_file_name + ".keras"),
classes=np.unique(y),
Expand Down
13 changes: 5 additions & 8 deletions aeon/clustering/deep_learning/_ae_fcn.py
Original file line number Diff line number Diff line change
Expand Up @@ -331,16 +331,13 @@ def get_test_params(cls, parameter_set="default"):
"batch_size": 4,
"use_bias": False,
"n_layers": 1,
"n_filters": 5,
"kernel_size": 3,
"n_filters": 4,
"kernel_size": 2,
"padding": "same",
"strides": 1,
"clustering_params": {
"distance": "euclidean",
"averaging_method": "mean",
"n_init": 1,
"max_iter": 30,
},
"latent_space_dim": 4,
"clustering_algorithm": "dummy",
"clustering_params": {"strategy": "random"},
}

return [param1]
13 changes: 6 additions & 7 deletions aeon/clustering/deep_learning/_ae_resnet.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class AEResNetClusterer(BaseDeepClusterer):
The number of convolution filters for all the convolution layers in the same
residual block, if not a list, the same number of filters is used in all
convolutions of all residual blocks.
kernel_sizes : int or list of int, default = [8, 5, 3]
kernel_size : int or list of int, default = [8, 5, 3]
The kernel size of all the convolution layers in one residual block, if not
a list, the same kernel size is used in all convolution layers.
strides : int or list of int, default = 1
Expand Down Expand Up @@ -352,12 +352,11 @@ def get_test_params(cls, parameter_set="default"):
"batch_size": 4,
"n_residual_blocks": 1,
"n_conv_per_residual_block": 1,
"clustering_params": {
"distance": "euclidean",
"averaging_method": "mean",
"n_init": 1,
"max_iter": 30,
},
"n_filters": 1,
"kernel_size": 2,
"use_bias": False,
"clustering_algorithm": "dummy",
"clustering_params": {"strategy": "random"},
}

test_params = [param]
Expand Down
9 changes: 7 additions & 2 deletions aeon/clustering/deep_learning/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from aeon.clustering._k_medoids import TimeSeriesKMedoids
from aeon.clustering._k_shapes import TimeSeriesKShapes
from aeon.clustering.base import BaseClusterer
from aeon.clustering.dummy import DummyClusterer


class BaseDeepClusterer(BaseClusterer, ABC):
Expand All @@ -17,7 +18,7 @@ class BaseDeepClusterer(BaseClusterer, ABC):
----------
n_clusters : int, default=None
Number of clusters for the deep learning model.
clustering_algorithm : str, {'kmeans', 'kshape', 'kmedoids'},
clustering_algorithm : str, {'kmeans', 'kshape', 'kmedoids', 'dummy'},
default="kmeans"
The clustering algorithm used in the latent space.
Options include:
Expand Down Expand Up @@ -115,7 +116,11 @@ def _fit_clustering(self, X):
else:
clustering_params_ = self.clustering_params
# clustering_params_["n_clusters"] = self.n_clusters
if self.clustering_algorithm == "kmeans":
if self.clustering_algorithm == "dummy":
self.clusterer = DummyClusterer(
n_clusters=self.n_clusters, **clustering_params_
)
elif self.clustering_algorithm == "kmeans":
Comment on lines +119 to +123
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this not just accept any BaseClusterer? Creating a useless option solely for testing is not a great way to resolve this IMO.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i wanted to change that for accepting an estimator input instead of string, but thought it might be a lot for the PR, but to keep the PR for testing purpose this can be done, if you think its ok to get all in one PR i dont mind can do the changes here

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't mind if you do it here. The dummy option is not a good addition IMO.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will add the changes then

self.clusterer = TimeSeriesKMeans(
n_clusters=self.n_clusters, **clustering_params_
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,49 +7,63 @@

from aeon.clustering import deep_learning
from aeon.testing.data_generation import make_example_3d_numpy
from aeon.utils.validation._dependencies import _check_soft_dependencies

__maintainer__ = ["hadifawaz1999"]

_deep_clr_classes = [
member[1] for member in inspect.getmembers(deep_learning, inspect.isclass)
]


@pytest.mark.skipif(
# not _check_soft_dependencies("tensorflow", severity="none"),
# See Issue #1761
True,
not _check_soft_dependencies("tensorflow", severity="none"),
reason="skip test if required soft dependency not available",
)
def test_random_state_deep_learning_clr():
@pytest.mark.parametrize("deep_clr", _deep_clr_classes)
def test_random_state_deep_learning_clr(deep_clr):
"""Test Deep Clusterer seeding."""
random_state = 42

X, _ = make_example_3d_numpy(random_state=random_state)
if not (deep_clr.__name__ in ["BaseDeepClusterer"]):
random_state = 42

deep_clr_classes = [
member[1] for member in inspect.getmembers(deep_learning, inspect.isclass)
]
X, _ = make_example_3d_numpy(random_state=random_state)

for i in range(len(deep_clr_classes)):
if "BaseDeepClusterer" in str(deep_clr_classes[i]):
continue
test_params = deep_clr.get_test_params()[0]
test_params["random_state"] = random_state

deep_clr1 = deep_clr_classes[i](
n_clusters=2, random_state=random_state, n_epochs=4
)
deep_clr1 = deep_clr(**test_params)
deep_clr1.fit(X)

layers1 = deep_clr1.training_model_.layers[1:]
encoder1 = deep_clr1.training_model_.layers[1]
decoder1 = deep_clr1.training_model_.layers[2]
encoder_layers1 = encoder1.layers[1:]
decoder_layers1 = decoder1.layers[1:]

deep_clr2 = deep_clr_classes[i](
n_clusters=2, random_state=random_state, n_epochs=4
)
deep_clr2 = deep_clr(**test_params)
deep_clr2.fit(X)

layers2 = deep_clr2.training_model_.layers[1:]
encoder2 = deep_clr2.training_model_.layers[1]
decoder2 = deep_clr2.training_model_.layers[2]
encoder_layers2 = encoder2.layers[1:]
decoder_layers2 = decoder2.layers[1:]

# test encoders
for i in range(len(encoder_layers1)):
weights1 = encoder_layers1[i].get_weights()
weights2 = encoder_layers2[i].get_weights()

assert len(weights1) == len(weights2)

for j in range(len(weights1)):
_weight1 = np.asarray(weights1[j])
_weight2 = np.asarray(weights2[j])

assert len(layers1) == len(layers2)
np.testing.assert_almost_equal(_weight1, _weight2, 4)

for i in range(len(layers1)):
weights1 = layers1[i].get_weights()
weights2 = layers2[i].get_weights()
# test decoders
for i in range(len(decoder_layers1)):
weights1 = decoder_layers1[i].get_weights()
weights2 = decoder_layers2[i].get_weights()

assert len(weights1) == len(weights2)

Expand Down
Loading