diff --git a/avisynth_filter/src/frame_handler.cpp b/avisynth_filter/src/frame_handler.cpp
index d8e5eff..7e6a6ba 100644
--- a/avisynth_filter/src/frame_handler.cpp
+++ b/avisynth_filter/src/frame_handler.cpp
@@ -165,31 +165,6 @@ auto FrameHandler::EndFlush(const std::function<void ()> &interim) -> void {
     Environment::GetInstance().Log(L"FrameHandler finish EndFlush()");
 }
 
-auto FrameHandler::Start() -> void {
-    _workerThread = std::thread(&FrameHandler::WorkerProc, this);
-}
-
-auto FrameHandler::Stop() -> void {
-    _isStopping = true;
-
-    BeginFlush();
-    EndFlush([]() -> void {
-        /*
-         * Stop the script after worker threads are paused and before flushing is done so that no new frame request (GetSourceFrame()) happens.
-         * And since _isFlushing is still on, existing frame request should also just drain instead of block.
-         *
-         * If no stop here, since AddInputSample() no longer adds frame, existing GetSourceFrame() calls will stuck forever.
-         */
-        MainFrameServer::GetInstance().StopScript();
-    });
-
-    if (_workerThread.joinable()) {
-        _workerThread.join();
-    }
-
-    _isStopping = false;
-}
-
 auto FrameHandler::ResetInput() -> void {
     _nextSourceFrameNb = 0;
     _maxRequestedFrameNb = 0;
diff --git a/avisynth_filter/src/frame_handler.h b/avisynth_filter/src/frame_handler.h
index 31f0a13..4b0bb55 100644
--- a/avisynth_filter/src/frame_handler.h
+++ b/avisynth_filter/src/frame_handler.h
@@ -12,6 +12,7 @@ class CSynthFilter;
 class FrameHandler {
 public:
     explicit FrameHandler(CSynthFilter &filter);
+    ~FrameHandler();
 
     DISABLE_COPYING(FrameHandler)
 
diff --git a/filter_common/src/filter.cpp b/filter_common/src/filter.cpp
index dbe2f4d..042aa11 100644
--- a/filter_common/src/filter.cpp
+++ b/filter_common/src/filter.cpp
@@ -29,11 +29,10 @@ CSynthFilter::CSynthFilter(LPUNKNOWN pUnk, HRESULT *phr)
 CSynthFilter::~CSynthFilter() {
     Environment::GetInstance().Log(L"Destroy CSynthFilter: %p", this);
 
-    _remoteControl.reset();
-    frameHandler.reset();
-
     _numFilterInstances -= 1;
     if (_numFilterInstances == 0) {
+        _remoteControl.reset();
+        frameHandler.reset();
         AuxFrameServer::Destroy();
         MainFrameServer::Destroy();
         FrameServerCommon::Destroy();
diff --git a/filter_common/src/frame_handler_common.cpp b/filter_common/src/frame_handler_common.cpp
index 7a45ad3..f3b0244 100644
--- a/filter_common/src/frame_handler_common.cpp
+++ b/filter_common/src/frame_handler_common.cpp
@@ -12,6 +12,34 @@ FrameHandler::FrameHandler(CSynthFilter &filter)
     ResetInput();
 }
 
+FrameHandler::~FrameHandler() {
+    if (_workerThread.joinable()) {
+        _isStopping = true;
+        Stop();
+        _workerThread.join();
+    }
+}
+
+auto FrameHandler::Start() -> void {
+    if (!_workerThread.joinable()) {
+        _isStopping = false;
+        _workerThread = std::thread(&FrameHandler::WorkerProc, this);
+    }
+}
+
+auto FrameHandler::Stop() -> void {
+    BeginFlush();
+    EndFlush([]() -> void {
+        /*
+    * Stop the script after worker threads are paused and before flushing is done so that no new frame request (GetSourceFrame()) happens.
+    * And since _isFlushing is still on, existing frame request should also just drain instead of block.
+    *
+    * If no stop here, since AddInputSample() no longer adds frame, existing GetSourceFrame() calls will stuck forever.
+    */
+        MainFrameServer::GetInstance().StopScript();
+    });
+}
+
 auto FrameHandler::GetInputBufferSize() const -> int {
     const std::shared_lock sharedSourceLock(_sourceMutex);
 
diff --git a/filter_common/src/input_pin.cpp b/filter_common/src/input_pin.cpp
index 8c37ce3..c8fb64c 100644
--- a/filter_common/src/input_pin.cpp
+++ b/filter_common/src/input_pin.cpp
@@ -91,7 +91,6 @@ auto CSynthFilterInputPin::Active() -> HRESULT {
 }
 
 auto CSynthFilterInputPin::Inactive() -> HRESULT {
-    _filter._remoteControl->Stop();
     _filter.frameHandler->Stop();
 
     return __super::Inactive();
diff --git a/filter_common/src/remote_control.cpp b/filter_common/src/remote_control.cpp
index f56425e..fd6efb6 100644
--- a/filter_common/src/remote_control.cpp
+++ b/filter_common/src/remote_control.cpp
@@ -13,16 +13,6 @@ RemoteControl::RemoteControl(CSynthFilter &filter)
 }
 
 RemoteControl::~RemoteControl() {
-    Stop();
-}
-
-auto RemoteControl::Start() -> void {
-    if (!_msgThread.joinable()) {
-        _msgThread = std::thread(&RemoteControl::Run, this);
-    }
-}
-
-auto RemoteControl::Stop() -> void {
     if (_hWnd != nullptr) {
         PostMessageW(_hWnd, WM_CLOSE, 0, 0);
     }
@@ -30,8 +20,12 @@ auto RemoteControl::Stop() -> void {
     if (_msgThread.joinable()) {
         _msgThread.join();
     }
+}
 
-    _hWnd = nullptr;
+auto RemoteControl::Start() -> void {
+    if (!_msgThread.joinable()) {
+        _msgThread = std::thread(&RemoteControl::Run, this);
+    }
 }
 
 auto RemoteControl::IsRunning() const -> bool {
diff --git a/filter_common/src/remote_control.h b/filter_common/src/remote_control.h
index b4489e6..0d92199 100644
--- a/filter_common/src/remote_control.h
+++ b/filter_common/src/remote_control.h
@@ -18,7 +18,6 @@ class RemoteControl {
     DISABLE_COPYING(RemoteControl)
 
     auto Start() -> void;
-    auto Stop() -> void;
     auto IsRunning() const -> bool;
 
 private:
diff --git a/vapoursynth_filter/src/frame_handler.cpp b/vapoursynth_filter/src/frame_handler.cpp
index a7c4cc8..6e99509 100644
--- a/vapoursynth_filter/src/frame_handler.cpp
+++ b/vapoursynth_filter/src/frame_handler.cpp
@@ -230,30 +230,6 @@ auto FrameHandler::EndFlush(const std::function<void ()> &interim) -> void {
     Environment::GetInstance().Log(L"FrameHandler finish EndFlush()");
 }
 
-auto FrameHandler::Start() -> void {
-    _isStopping = false;
-    _workerThread = std::thread(&FrameHandler::WorkerProc, this);
-}
-
-auto FrameHandler::Stop() -> void {
-    _isStopping = true;
-
-    BeginFlush();
-    EndFlush([]() -> void {
-        /*
-         * Stop the script after worker threads are paused and before flushing is done so that no new frame request (GetSourceFrame()) happens.
-         * And since _isFlushing is still on, existing frame request should also just drain instead of block.
-         *
-         * If no stop here, since AddInputSample() no longer adds frame, existing GetSourceFrame() calls will stuck forever.
-         */
-        MainFrameServer::GetInstance().StopScript();
-    });
-
-    if (_workerThread.joinable()) {
-        _workerThread.join();
-    }
-}
-
 FrameHandler::SourceFrameInfo::~SourceFrameInfo() {
     AVSF_VS_API->freeFrame(frame);
 }
diff --git a/vapoursynth_filter/src/frame_handler.h b/vapoursynth_filter/src/frame_handler.h
index 71343e4..661056c 100644
--- a/vapoursynth_filter/src/frame_handler.h
+++ b/vapoursynth_filter/src/frame_handler.h
@@ -12,6 +12,7 @@ class CSynthFilter;
 class FrameHandler {
 public:
     explicit FrameHandler(CSynthFilter &filter);
+    ~FrameHandler();
 
     DISABLE_COPYING(FrameHandler)