diff --git a/ivy/stateful/converters.py b/ivy/stateful/converters.py
index 13b5e5cfde568..599b5532d0a24 100644
--- a/ivy/stateful/converters.py
+++ b/ivy/stateful/converters.py
@@ -527,10 +527,12 @@ def call(self, *args, training=None, **kwargs):
                 return ret
 
             def __call__(self, *args, **kwargs):
-                ivy.set_backend("tensorflow")
-                args, kwargs = ivy.args_to_new_backend(*args, native=True, **kwargs)
-                ivy.previous_backend()
-
+                if ivy.backend != "tensorflow":
+                    ivy.set_backend("tensorflow")
+                    args, kwargs = ivy.args_to_new_backend(*args, native=True, **kwargs)
+                    ivy.previous_backend()
+                else:
+                    args, kwargs = ivy.args_to_new_backend(*args, native=True, **kwargs)
                 return super(KerasModel, self).__call__(*args, **kwargs)
 
             def to_device(self, device):