From d1d21e742563a7df087692f7d5b3d60c0f0e5829 Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarithatuka@gmail.com>
Date: Tue, 5 Dec 2017 09:25:00 -0500
Subject: [PATCH 1/5] DNM osd: add 'ceph [tell|daemon] osd.id smart'

Also added 'ceph daemon osd.id list_devices' which prints to stdout the
osd devices. 'ceph [tell|daemon] osd.id smart' probes the osd devices
for SMART data and prints it to stdout in a JSON format. It assumes smartctl '--json' feature
exists.

Signed-off-by: Yaarit Hatuka <yaarithatuka@gmail.com>
---
 src/common/options.cc            |   4 ++
 src/osd/OSD.cc                   | 107 +++++++++++++++++++++++++++++++
 src/osd/OSD.h                    |   3 +
 src/pybind/mgr/smart/__init__.py |   2 +
 src/pybind/mgr/smart/module.py   |  35 ++++++++++
 5 files changed, 151 insertions(+)
 create mode 100644 src/pybind/mgr/smart/__init__.py
 create mode 100644 src/pybind/mgr/smart/module.py

diff --git a/src/common/options.cc b/src/common/options.cc
index b286457b9b35f..ee73aa106b9fb 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1632,6 +1632,10 @@ std::vector<Option> get_global_options() {
     .set_min(2)
     .set_description("Number of striping periods to zero head of MDS journal write position"),
 
+     Option("osd_smart_report_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(5)
+    .set_description("Timeout (in seconds) for smarctl to run, default is set to 5"),
+
     Option("osd_check_max_object_name_len_on_startup", Option::TYPE_BOOL, Option::LEVEL_DEV)
     .set_default(true)
     .set_description(""),
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index bf613d58d2602..afa979e93a0fb 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -46,6 +46,7 @@
 #include "common/version.h"
 #include "common/io_priority.h"
 #include "common/pick_address.h"
+#include "common/SubProcess.h"
 
 #include "os/ObjectStore.h"
 #ifdef HAVE_LIBFUSE
@@ -146,6 +147,9 @@
 #include "common/config.h"
 #include "common/EventTrace.h"
 
+#include "json_spirit/json_spirit_reader.h"
+#include "json_spirit/json_spirit_writer.h"
+
 #ifdef WITH_LTTNG
 #define TRACEPOINT_DEFINE
 #define TRACEPOINT_PROBE_DYNAMIC_LINKAGE
@@ -2349,6 +2353,19 @@ will start to track new ops received afterwards.";
     set<int> poollist = get_mapped_pools();
     f->dump_stream("pool_list") << poollist;
     f->close_section();
+  } else if (admin_command == "smart") {
+    probe_smart(ss);
+  } else if (admin_command == "list_devices") {
+    set<string> devnames;
+    store->get_devices(&devnames);
+    f->open_object_section("list_devices");
+    for (auto dev : devnames) {
+      f->dump_string("device", "/dev/" + dev);
+      if (dev.find("dm-") == 0) {
+	continue;
+      }
+    }
+    f->close_section();
   } else {
     assert(0 == "broken asok registration");
   }
@@ -2881,6 +2898,18 @@ void OSD::final_init()
                                      asok_hook,
                                      "dump pools whose PG(s) are mapped to this OSD.");
 
+  assert(r == 0);
+
+  r = admin_socket->register_command("smart", "smart",
+                                     asok_hook,
+                                     "probe OSD devices for SMART data.");
+
+  assert(r == 0);
+
+  r = admin_socket->register_command("list_devices", "list_devices",
+                                     asok_hook,
+                                     "list OSD devices.");
+
   test_ops_hook = new TestOpsSocketHook(&(this->service), this->store);
   // Note: pools are CephString instead of CephPoolname because
   // these commands traditionally support both pool names and numbers
@@ -3367,6 +3396,8 @@ int OSD::shutdown()
   cct->get_admin_socket()->unregister_command("dump_pgstate_history");
   cct->get_admin_socket()->unregister_command("compact");
   cct->get_admin_socket()->unregister_command("get_mapped_pools");
+  cct->get_admin_socket()->unregister_command("smart");
+  cct->get_admin_socket()->unregister_command("list_devices");
   delete asok_hook;
   asok_hook = NULL;
 
@@ -6043,6 +6074,9 @@ COMMAND("compact",
         "compact object store's omap. "
         "WARNING: Compaction probably slows your requests",
         "osd", "rw", "cli,rest")
+COMMAND("smart",
+        "runs smartctl on this osd devices.  ",
+        "osd", "rw", "cli,rest")
 };
 
 void OSD::do_command(Connection *con, ceph_tid_t tid, vector<string>& cmd, bufferlist& data)
@@ -6457,6 +6491,10 @@ void OSD::do_command(Connection *con, ceph_tid_t tid, vector<string>& cmd, buffe
     ss << "compacted omap in " << duration << " seconds";
   }
 
+  else if (prefix == "smart") {
+    probe_smart(ds);
+  }
+
   else {
     ss << "unrecognized command! " << cmd;
     r = -EINVAL;
@@ -6475,6 +6513,75 @@ void OSD::do_command(Connection *con, ceph_tid_t tid, vector<string>& cmd, buffe
   }
 }
 
+void OSD::probe_smart(ostream& ss)
+{
+  set<string> devnames;
+  store->get_devices(&devnames);
+  uint64_t smart_timeout = cct->_conf->get_val<uint64_t>("osd_smart_report_timeout");
+  std::string result;
+
+  json_spirit::mObject json_map; // == typedef std::map<std::string, mValue> mObject;
+  json_spirit::mValue smart_json;
+
+  for (auto dev : devnames) {
+      // smartctl works only on physical devices; filter out any logical device
+      if (dev.find("dm-") == 0) {
+	  continue;
+      }
+
+      if (probe_smart_device(("/dev/" + dev).c_str(), smart_timeout, &result)) {
+	  derr << "probe_smart_device failed for /dev/" << dev << ", continuing to next device"<< dendl;
+	  continue;
+      }
+
+      // TODO: change to read_or_throw?
+      if (!json_spirit::read(result, smart_json)) {
+	  derr << "smartctl JSON output of /dev/" + dev + " is invalid" << dendl;
+      }
+      else { //json is valid, assigning
+	  json_map[dev] = smart_json;
+      }
+      // no need to result.clear() or clear smart_json
+  }
+  json_spirit::write(json_map, ss, json_spirit::pretty_print);
+}
+
+int OSD::probe_smart_device(const char *device, int timeout, std::string *result)
+{
+  // when using --json, smartctl will report its errors in JSON format to stdout 
+  SubProcessTimed smartctl("sudo", SubProcess::CLOSE, SubProcess::PIPE, SubProcess::CLOSE, timeout);
+  smartctl.add_cmd_args(
+      "smartctl",
+      "-a",
+      //"-x",
+      "--json",
+      device,
+      NULL);
+
+  int ret = smartctl.spawn();
+  if (ret != 0) {
+    derr << "failed run smartctl: " << smartctl.err() << dendl;
+    return ret;
+  }
+
+  bufferlist output;
+  ret = output.read_fd(smartctl.get_stdout(), 100*1024);
+  if (ret < 0) {
+    derr << "failed read from smartctl: " << cpp_strerror(-ret) << dendl;
+    return ret;
+  }
+
+  derr << "smartctl output is: " << output.c_str() << dendl;
+  *result = output.c_str(); 
+
+  if (smartctl.join() != 0) {
+    derr << smartctl.err() << dendl;
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
 bool OSD::heartbeat_dispatch(Message *m)
 {
   dout(30) << "heartbeat_dispatch " << m << dendl;
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 07eb73a81a0ce..e18fc10f275c8 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -2349,6 +2349,9 @@ class OSD : public Dispatcher,
 
   float get_osd_recovery_sleep();
 
+  void probe_smart(ostream& ss);
+  int probe_smart_device(const char *device, int timeout, std::string *result);
+
 public:
   static int peek_meta(ObjectStore *store, string& magic,
 		       uuid_d& cluster_fsid, uuid_d& osd_fsid, int& whoami);
diff --git a/src/pybind/mgr/smart/__init__.py b/src/pybind/mgr/smart/__init__.py
new file mode 100644
index 0000000000000..79f5b86fd5045
--- /dev/null
+++ b/src/pybind/mgr/smart/__init__.py
@@ -0,0 +1,2 @@
+
+from module import *  # NOQA
diff --git a/src/pybind/mgr/smart/module.py b/src/pybind/mgr/smart/module.py
new file mode 100644
index 0000000000000..8b2d54c176776
--- /dev/null
+++ b/src/pybind/mgr/smart/module.py
@@ -0,0 +1,35 @@
+
+"""
+Pulling smart data from OSD
+"""
+
+import json
+from mgr_module import MgrModule, CommandResult
+
+
+class Module(MgrModule):
+    COMMANDS = [
+        {
+            "cmd": "osd smart get "
+                   "name=osd_id,type=CephString,req=true",
+            "desc": "Get smart data for osd.id",
+            "perm": "r"
+        },
+    ]
+
+    def handle_command(self, cmd):
+        self.log.error("handle_command")
+
+        if cmd['prefix'] == 'osd smart get':
+            result = CommandResult('')
+            self.send_command(result, 'osd', cmd['osd_id'], json.dumps({
+                'prefix': 'smart',
+                'format': 'json',
+            }), '')
+            r, outb, outs = result.wait()
+            return (r, outb, outs)
+
+        else:
+            # mgr should respect our self.COMMANDS and not call us for
+            # any prefix we don't advertise
+            raise NotImplementedError(cmd['prefix'])

From 494764f3d782324aefdb2f32e85570274d965c4d Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarithatuka@gmail.com>
Date: Wed, 17 Jan 2018 22:16:17 -0500
Subject: [PATCH 2/5] DNM: pybind/mgr: add smart mgr module

This mgr module contains two commands:
- 'osd smart get <osd.id>' # queries a specific OSD for SMART data of all
of its disks.
- 'osd smart get all' # queries all OSDs in the mgr's osd_map for SMART
  data.
The mgr invokes a 'tell' command on the OSD side, which scrapes it for
SMART metrics.

Signed-off-by: Yaarit Hatuka <yaarithatuka@gmail.com>
---
 src/pybind/mgr/smart/module.py | 41 +++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/src/pybind/mgr/smart/module.py b/src/pybind/mgr/smart/module.py
index 8b2d54c176776..a4b88fff8513f 100644
--- a/src/pybind/mgr/smart/module.py
+++ b/src/pybind/mgr/smart/module.py
@@ -1,6 +1,6 @@
 
 """
-Pulling smart data from OSD
+Pulling SMART data from OSD
 """
 
 import json
@@ -12,23 +12,48 @@ class Module(MgrModule):
         {
             "cmd": "osd smart get "
                    "name=osd_id,type=CephString,req=true",
+            "desc": "Get smart data for osd_id",
+            "perm": "r"
+        },
+        {
+            "cmd": "osd smart get all",
             "desc": "Get smart data for osd.id",
             "perm": "r"
         },
     ]
 
+    def handle_osd_smart_get(self, osd_id):
+        result = CommandResult('')
+        self.send_command(result, 'osd', osd_id, json.dumps({
+            'prefix': 'smart',
+            'format': 'json',
+        }), '')
+        r, outb, outs = result.wait()
+
+        return (r, outb, outs)
+
     def handle_command(self, cmd):
         self.log.error("handle_command")
 
-        if cmd['prefix'] == 'osd smart get':
-            result = CommandResult('')
-            self.send_command(result, 'osd', cmd['osd_id'], json.dumps({
-                'prefix': 'smart',
-                'format': 'json',
-            }), '')
-            r, outb, outs = result.wait()
+        if cmd['prefix'] == "osd smart get":
+            r, outb, outs = self.handle_osd_smart_get(cmd['osd_id'])
+
             return (r, outb, outs)
 
+        elif cmd['prefix'] == "osd smart get all":
+            osdmap = self.get("osd_map")
+            assert osdmap is not None
+
+            smart_map = {}
+            # TODO what do we do in case of many osds? should we query all osds concurrently?
+            for osd in osdmap['osds']:
+                osd_id = osd['osd']
+                r, outb, outs = self.handle_osd_smart_get(str(osd_id))
+                smart_map[osd_id] = json.loads(outb)
+
+            # TODO do we need to keep the other r and outs values?
+            return (r, json.dumps(smart_map), outs)
+
         else:
             # mgr should respect our self.COMMANDS and not call us for
             # any prefix we don't advertise

From fc57a63c9bb46c79060398872553422d992a3b18 Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarithatuka@gmail.com>
Date: Mon, 12 Feb 2018 08:52:20 -0500
Subject: [PATCH 3/5] src/pybind/mgr added scrape commands.

---
 src/pybind/mgr/smart/module.py | 109 +++++++++++++++++++++++++++------
 1 file changed, 90 insertions(+), 19 deletions(-)

diff --git a/src/pybind/mgr/smart/module.py b/src/pybind/mgr/smart/module.py
index a4b88fff8513f..238d9f6ef3246 100644
--- a/src/pybind/mgr/smart/module.py
+++ b/src/pybind/mgr/smart/module.py
@@ -5,54 +5,125 @@
 
 import json
 from mgr_module import MgrModule, CommandResult
+import rados
+import time
 
+OBJECT_NAME = "mytest"
+ITER = 2
+pool_name = "smart_data"
 
 class Module(MgrModule):
     COMMANDS = [
+        {
+            "cmd": "osd smart scrape "
+                   "name=osd_id,type=CephString,req=true",
+            "desc": "Scraping SMART data of osd<osd_id>",
+            "perm": "r"
+        },
+        {
+            "cmd": "osd smart scrape all",
+            "desc": "Scraping SMART data of all OSDs",
+            "perm": "r"
+        },
         {
             "cmd": "osd smart get "
                    "name=osd_id,type=CephString,req=true",
-            "desc": "Get smart data for osd_id",
+            "desc": "Get SMART data for osd_id",
             "perm": "r"
         },
         {
             "cmd": "osd smart get all",
-            "desc": "Get smart data for osd.id",
+            "desc": "Get SMART data for all OSDs",
+            "perm": "r"
+        },
+        {
+            "cmd": "pool test",
+            "desc": "Just a pool test",
             "perm": "r"
         },
     ]
 
-    def handle_osd_smart_get(self, osd_id):
+    def open_connection(self):
+        # looking for the right pool
+        # TODO open pool, and if it fails - create and open
+        pools = self.rados.list_pools()
+        is_pool = False
+        for pool in pools:
+            self.log.debug('pool name is %s' % pool)
+            if pool == pool_name:
+                is_pool = True
+                break
+
+        # TODO make pool name configurable
+        if not is_pool:
+            self.rados.create_pool(pool_name)
+            self.log.debug('create %s pool' % pool_name)
+
+        ioctx = self.rados.open_ioctx(pool_name)
+
+        return (ioctx)
+
+    def handle_osd_smart_scrape_all(self):
+        osdmap = self.get("osd_map")
+        assert osdmap is not None
+
+        ioctx = self.open_connection()
+
+        for osd in osdmap['osds']:
+            osd_id = osd['osd']
+            self.do_osd_smart_scrape(str(osd_id), ioctx)
+
+        ioctx.close()
+        return (0, "", "")
+
+    def handle_osd_smart_scrape(self, osd_id):
+        ioctx = self.open_connection()
+        self.do_osd_smart_scrape(osd_id, ioctx);
+        ioctx.close()
+        return (0, "", "")
+
+    def do_osd_smart_scrape(self, osd_id, ioctx):
+        # running 'tell' command on OSD, retrieving smartctl results
         result = CommandResult('')
         self.send_command(result, 'osd', osd_id, json.dumps({
             'prefix': 'smart',
             'format': 'json',
         }), '')
         r, outb, outs = result.wait()
+        smart_data = json.load(bytes(outb))
+
+        osd_host = self.get_metadata('osd', osd_id)['hostname']
+        self.log.debug('osd host %s' % osd_host)
+
+        for device, device_smart_data in smart_data.items():
+            object_name = osd_host + ":" + device
+            with rados.WriteOpCtx() as op:
+                # TODO should we change second arg to be byte?
+                # TODO what about the flag in ioctx.operate_write_op()?
+                now = str(time.time())
+                ioctx.set_omap(op, now, (str(json.dumps(device_smart_data)),))
+                ioctx.operate_write_op(op, object_name)
+                self.log.debug('writing object %s, key: %s, value: %s' % object_name, now, str(json.dumps(device_smart_data)))
 
-        return (r, outb, outs)
+        return (0, "do osd scrape", "test")
 
     def handle_command(self, cmd):
         self.log.error("handle_command")
 
-        if cmd['prefix'] == "osd smart get":
-            r, outb, outs = self.handle_osd_smart_get(cmd['osd_id'])
+        if cmd['prefix'] == "osd smart scrape":
+            return self.handle_osd_smart_scrape(cmd['osd_id'])
 
-            return (r, outb, outs)
+        elif cmd['prefix'] == "osd smart scrape all":
+            return self.handle_osd_smart_scrape_all()
+
+        elif cmd['prefix'] == "osd smart get":
+            return (0, "get", "test")
 
         elif cmd['prefix'] == "osd smart get all":
-            osdmap = self.get("osd_map")
-            assert osdmap is not None
-
-            smart_map = {}
-            # TODO what do we do in case of many osds? should we query all osds concurrently?
-            for osd in osdmap['osds']:
-                osd_id = osd['osd']
-                r, outb, outs = self.handle_osd_smart_get(str(osd_id))
-                smart_map[osd_id] = json.loads(outb)
-
-            # TODO do we need to keep the other r and outs values?
-            return (r, json.dumps(smart_map), outs)
+            return (0, "get all", "test")
+
+        elif cmd['prefix'] == "pool test":
+            return (0, "pool", "test")
 
         else:
             # mgr should respect our self.COMMANDS and not call us for

From bb7e61f232aede4ed5cc360528fcda6176bf8dfe Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarithatuka@gmail.com>
Date: Mon, 12 Feb 2018 09:38:59 -0500
Subject: [PATCH 4/5] pybind/mgr: fixed json errors.

---
 src/pybind/mgr/smart/module.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/pybind/mgr/smart/module.py b/src/pybind/mgr/smart/module.py
index 238d9f6ef3246..d412dd1baa225 100644
--- a/src/pybind/mgr/smart/module.py
+++ b/src/pybind/mgr/smart/module.py
@@ -12,6 +12,7 @@
 ITER = 2
 pool_name = "smart_data"
 
+
 class Module(MgrModule):
     COMMANDS = [
         {
@@ -78,7 +79,7 @@ def handle_osd_smart_scrape_all(self):
 
     def handle_osd_smart_scrape(self, osd_id):
         ioctx = self.open_connection()
-        self.do_osd_smart_scrape(osd_id, ioctx);
+        self.do_osd_smart_scrape(osd_id, ioctx)
         ioctx.close()
         return (0, "", "")
 
@@ -90,7 +91,7 @@ def do_osd_smart_scrape(self, osd_id, ioctx):
             'format': 'json',
         }), '')
         r, outb, outs = result.wait()
-        smart_data = json.load(bytes(outb))
+        smart_data = json.loads(outb)
 
         osd_host = self.get_metadata('osd', osd_id)['hostname']
         self.log.debug('osd host %s' % osd_host)
@@ -101,9 +102,9 @@ def do_osd_smart_scrape(self, osd_id, ioctx):
                 # TODO should we change second arg to be byte?
                 # TODO what about the flag in ioctx.operate_write_op()?
                 now = str(time.time())
-                ioctx.set_omap(op, now, (str(json.dumps(device_smart_data)),))
+                ioctx.set_omap(op, (now,), (str(json.dumps(device_smart_data)),))
                 ioctx.operate_write_op(op, object_name)
-                self.log.debug('writing object %s, key: %s, value: %s' % object_name, now, str(json.dumps(device_smart_data)))
+                self.log.debug('writing object %s, key: %s, value: %s' % (object_name, now, str(json.dumps(device_smart_data))))
 
         return (0, "do osd scrape", "test")
 

From b35f62b32ec2a3e49a4849f3861703931349b925 Mon Sep 17 00:00:00 2001
From: Yaarit Hatuka <yaarithatuka@gmail.com>
Date: Wed, 28 Feb 2018 14:47:37 -0500
Subject: [PATCH 5/5] adding SMART cmds to mgr.

---
 src/pybind/mgr/smart/module.py | 311 ++++++++++++++++++++++++++++-----
 1 file changed, 268 insertions(+), 43 deletions(-)

diff --git a/src/pybind/mgr/smart/module.py b/src/pybind/mgr/smart/module.py
index d412dd1baa225..011b9054fa221 100644
--- a/src/pybind/mgr/smart/module.py
+++ b/src/pybind/mgr/smart/module.py
@@ -6,15 +6,34 @@
 import json
 from mgr_module import MgrModule, CommandResult
 import rados
-import time
+from threading import Event
+from datetime import datetime, timedelta, date, time
 
-OBJECT_NAME = "mytest"
-ITER = 2
-pool_name = "smart_data"
+TIME_FORMAT = '%Y%m%d-%H%M%S'
+SCRAPE_FREQUENCY = 86400  # seconds
+
+# TODO maybe have a 'refresh' command to run after config-key set
+# to wake the sleep interval
+# TODO document
 
 
 class Module(MgrModule):
     COMMANDS = [
+        {
+            "cmd": "smart status",
+            "desc": "Show smart status",
+            "perm": "r",
+        },
+        {
+            "cmd": "smart on",
+            "desc": "Enable automatic SMART scraping",
+            "perm": "rw",
+        },
+        {
+            "cmd": "smart off",
+            "desc": "Disable automatic SMART scraping",
+            "perm": "rw",
+        },
         {
             "cmd": "osd smart scrape "
                    "name=osd_id,type=CephString,req=true",
@@ -22,31 +41,46 @@ class Module(MgrModule):
             "perm": "r"
         },
         {
-            "cmd": "osd smart scrape all",
+            "cmd": "osd smart scrape-all",
             "desc": "Scraping SMART data of all OSDs",
             "perm": "r"
         },
         {
-            "cmd": "osd smart get "
-                   "name=osd_id,type=CephString,req=true",
-            "desc": "Get SMART data for osd_id",
+            "cmd": "osd smart dump "
+                   "name=obj_name,type=CephString,req=true",
+            "desc": "Get SMART data of osd_id, which was scraped and stored in a rados object",
             "perm": "r"
         },
         {
-            "cmd": "osd smart get all",
-            "desc": "Get SMART data for all OSDs",
+            "cmd": "osd smart dump-all",
+            "desc": "Get SMART data of all OSDs, which was scraped and stored in a rados object",
             "perm": "r"
         },
         {
-            "cmd": "pool test",
-            "desc": "Just a pool test",
-            "perm": "r"
+            "cmd": "smart predict-failure",
+            "desc": "Run a failure prediction model based on the latest SMART stats.",
+            "perm": "rw",
+        },
+        {
+            "cmd": "smart warn",
+            "desc": "Test warning.",
+            "perm": "rw",
         },
     ]
+    active = False
+    run = True
+    event = Event()
+    # pool_name (check if it makes sense to leave here?)
+    # scrape_frequency = SCRAPE_FREQUENCY
+    # failure_prediction_action = 'warn'
+    # failure_prediction_model = 'trivial'
+    # last_scrape = 000000
+    # dump_data = {}
 
     def open_connection(self):
+        # TODO check that pool name is actually configurable
+        pool_name = self.get_config('pool_name') or 'smart_data'
         # looking for the right pool
-        # TODO open pool, and if it fails - create and open
         pools = self.rados.list_pools()
         is_pool = False
         for pool in pools:
@@ -54,29 +88,12 @@ def open_connection(self):
             if pool == pool_name:
                 is_pool = True
                 break
-
-        # TODO make pool name configurable
         if not is_pool:
             self.rados.create_pool(pool_name)
             self.log.debug('create %s pool' % pool_name)
-
         ioctx = self.rados.open_ioctx(pool_name)
-
         return (ioctx)
 
-    def handle_osd_smart_scrape_all(self):
-        osdmap = self.get("osd_map")
-        assert osdmap is not None
-
-        ioctx = self.open_connection()
-
-        for osd in osdmap['osds']:
-            osd_id = osd['osd']
-            self.do_osd_smart_scrape(str(osd_id), ioctx)
-
-        ioctx.close()
-        return (0, "", "")
-
     def handle_osd_smart_scrape(self, osd_id):
         ioctx = self.open_connection()
         self.do_osd_smart_scrape(osd_id, ioctx)
@@ -91,40 +108,248 @@ def do_osd_smart_scrape(self, osd_id, ioctx):
             'format': 'json',
         }), '')
         r, outb, outs = result.wait()
+        # TODO add try-catch block for json
         smart_data = json.loads(outb)
 
         osd_host = self.get_metadata('osd', osd_id)['hostname']
-        self.log.debug('osd host %s' % osd_host)
+        if not osd_host:
+            self.log.debug('cannot get osd hostname of osd_id: %s' % osd_id)
+            return
+        self.log.debug('osd hostname %s' % osd_host)
 
         for device, device_smart_data in smart_data.items():
             object_name = osd_host + ":" + device
             with rados.WriteOpCtx() as op:
                 # TODO should we change second arg to be byte?
                 # TODO what about the flag in ioctx.operate_write_op()?
-                now = str(time.time())
+                now = time.strftime(TIME_FORMAT)
                 ioctx.set_omap(op, (now,), (str(json.dumps(device_smart_data)),))
                 ioctx.operate_write_op(op, object_name)
                 self.log.debug('writing object %s, key: %s, value: %s' % (object_name, now, str(json.dumps(device_smart_data))))
 
-        return (0, "do osd scrape", "test")
+    def handle_osd_smart_scrape_all(self):
+        osdmap = self.get("osd_map")
+        assert osdmap is not None
+        ioctx = self.open_connection()
+        for osd in osdmap['osds']:
+            osd_id = osd['osd']
+            self.log.debug('scraping osd %s for smart data' % str(osd_id))
+            self.do_osd_smart_scrape(str(osd_id), ioctx)
+        ioctx.close()
+        return (0, "", "")
+
+    # TODO add changes here - avoid duplication
+    def predict_failure(self, model, action):
+        ioctx = self.open_connection()
+        object_iterator = ioctx.list_objects()
+        while True:
+            try:
+                rados_object = object_iterator.next()
+                obj_name = rados_object.key
+                self.log.debug('object name = %s' % rados_object.key)
+                self.do_osd_smart_dump(obj_name, ioctx)
+            except StopIteration:
+                self.log.debug('iteration stopped')
+                break
+        ioctx.close()
+        return
+
+    def shutdown(self):
+        self.log.info('Stopping')
+        self.run = False
+        self.event.set()
+
+    # TODO maybe change 'begin' to be datetime object instead of str
+    def seconds_to_next_run(self, begin, interval):
+        now = datetime.now()
+        # creating datetime object for today's first run
+        today = date.today()
+        begin_time = time(int(begin[0:2]), int(begin[2:4]))
+        begin_time_today = datetime.combine(today, begin_time)
+        # In case we missed today's first run - rewinding the clock a day,
+        # and calculating the next run by adding the seconds interval.
+        # This assumes a minimum of one daily run.
+        if begin_time_today > now:
+            begin_time_today -= timedelta(days=1)
+        while begin_time_today < now:
+            begin_time_today += timedelta(seconds=interval)
+        delta = begin_time_today - now
+        return int(delta.total_seconds())
+
+    def serve(self):
+        self.log.info("Starting")
+        while self.run:
+            # TODO see if there might be a bug if active == false
+            self.active = self.get_config('active', '') is not ''
+            # TODO convert begin string to a datetime object
+            begin_time = self.get_config('begin_time') or '0000'
+            scrape_frequency = self.get_config('scrape_frequency') or SCRAPE_FREQUENCY
+            failure_prediction_model = self.get_config('failure_prediction_model ') or 'trivial'
+            failure_prediction_action = self.get_config('failure_prediction_action ') or 'warn'
+            timeofday = time.strftime('%H%M%S', time.localtime())
+
+            sleep_interval = seconds_to_next_run(begin_time, scrape_frequency)
+            self.log.debug('Sleeping for %d', sleep_interval)
+            ret = self.event.wait(sleep_interval)
+            self.event.clear()
+
+            # in case 'wait' was interrupted, it could mean config was changed
+            # by the user; go back and read config vars
+            if ret:
+                continue
+            # in case we waited the entire interval
+            else:
+                # TODO specify the exact time of next run
+                self.log.debug('Waking up [%s, scheduled for %s, now %s]',
+                               "active" if self.active else "inactive",
+                               begin_time, timeofday)
+                if self.active:
+                    self.log.debug('Running')
+                    self.handle_osd_smart_scrape_all()
+                    self.predict_failure(failure_prediction_model, failure_prediction_action)
+
+    # TODO remove
+    def get_smart_attr_val(self, table, t_id):
+        for item in table:
+            self.log.debug('desired table id is %s and curr val is: %s' % (t_id, item['id']))
+            self.log.debug('item type is: %s' % type(item))
+            self.log.debug('table type is: %s' % type(table))
+            if item['id'] == t_id:
+                return item
+        return None
+
+    # TODO remove
+    def get_lastest_scrape_date(self, ioctx, obj_name):
+        with rados.ReadOpCtx() as read_op:
+            iter, ret = ioctx.get_omap_keys(read_op, "", -1)
+            assert(ret == 0)
+            ioctx.operate_read_op(read_op, obj_name)
+            latest = 0
+            for date, none in list(iter):
+                date = str(date)
+                if date > latest:
+                    latest = date
+        return latest
+
+    # TODO remove
+    def get_scrape_of_date(self, ioctx, obj_name, date):
+        scrape_dump = 0
+        with rados.ReadOpCtx() as read_op:
+            iter, ret = ioctx.get_omap_vals_by_keys(read_op, (date,))
+            assert(ret == 0)
+            ioctx.operate_read_op(read_op, obj_name)
+            for (k, v) in list(iter):
+                # self.log.debug("date requested is: %s and value is: %s" % (k, v))
+                scrape_dump = v
+        return scrape_dump
+
+    # TODO maybe add a 'date' param, for 'start key' in get_omap_vals
+    def do_osd_smart_dump(self, obj_name, ioctx):
+        """
+        date = self.get_lastest_scrape_date(ioctx, obj_name)
+        self.log.debug("hello___: %s" % date)
+        # TODO check that date exist
+        scrape_dump = self.get_scrape_of_date(ioctx, obj_name, date)
+        self.log.debug("hello wow val is: %s" % scrape_dump)
+        """
+        smart_dump = {}
+        with rados.ReadOpCtx() as op:
+            # TODO ask from a specific key (specific date)
+            iter, ret = ioctx.get_omap_vals(op, "", "", -1)
+            assert(ret == 0)
+            ioctx.operate_read_op(op, obj_name)
+            # k == date; v == json_blob
+            for (k, v) in list(iter):
+                self.log.debug('Dumping SMART data of object: %s, key: %s, val %s' % (obj_name, k, v))
+                try:
+                    smart_dump[k] = json.loads(str(v))
+                    # TODO move this
+                    """
+                    table = json.loads(str(v))
+                    self.log.debug('ata version is: %s' % table['ata_version'])
+                    self.log.debug('ata_smart_attributes are: %s' % table['ata_smart_attributes']['table'][0]['id'])
+                    self.log.debug('calling table func: ')
+                    res = self.get_smart_attr_val(table['ata_smart_attributes']['table'], 5)
+                    self.log.debug("result from table read is: %s" % res)
+                    self.log.debug("value is read is: %s" % res['value'])
+                    res = self.get_smart_attr_val(table['ata_smart_attributes']['table'], 187)
+                    self.log.debug("value of read is: %s --------" % res['value'])
+                    """
+                except:
+                    # TODO get the full error message
+                    self.log.debug('Probably invalid or empty JSON')
+        return str(json.dumps(smart_dump))
+
+    def handle_osd_smart_dump(self, cmd):
+        ioctx = self.open_connection()
+        smart_dump = self.do_osd_smart_dump(cmd['obj_name'], ioctx)
+        ioctx.close()
+        return (0, "", str(smart_dump))
+
+    def handle_osd_smart_dump_all(self):
+        ioctx = self.open_connection()
+        object_iterator = ioctx.list_objects()
+        # TODO see whether to change return val
+        smart_dump = {}
+        while True:
+            try:
+                rados_object = object_iterator.next()
+                obj_name = rados_object.key
+                self.log.debug('object name = %s' % rados_object.key)
+                smart_dump[obj_name] = self.do_osd_smart_dump(obj_name, ioctx)
+            except StopIteration:
+                self.log.debug('iteration stopped')
+                break
+        ioctx.close()
+        # TODO maybe have a wrapper for the cmd call
+        # and return 0 or 1 here (and json obj)
+        return (0, "", str(smart_dump))
 
     def handle_command(self, cmd):
-        self.log.error("handle_command")
+        self.log.info("Handling command: '%s'" % str(cmd))
+
+        if cmd['prefix'] == 'smart status':
+            s = {
+                'active': self.active,
+            }
+            return (0, json.dumps(s, indent=4), '')
+
+        elif cmd['prefix'] == 'smart on':
+            if not self.active:
+                self.set_config('active', '1')
+                self.active = True
+            self.event.set()
+            return (0, '', '')
+
+        elif cmd['prefix'] == 'smart off':
+            if self.active:
+                self.set_config('active', '')
+                self.active = False
+            self.event.set()
+            return (0, '', '')
 
-        if cmd['prefix'] == "osd smart scrape":
+        elif cmd['prefix'] == "osd smart scrape":
             return self.handle_osd_smart_scrape(cmd['osd_id'])
 
-        elif cmd['prefix'] == "osd smart scrape all":
+        elif cmd['prefix'] == "osd smart scrape-all":
             return self.handle_osd_smart_scrape_all()
 
-        elif cmd['prefix'] == "osd smart get":
-            return (0, "get", "test")
+        elif cmd['prefix'] == "osd smart dump":
+            return self.handle_osd_smart_dump(cmd)
 
-        elif cmd['prefix'] == "osd smart get all":
-            return (0, "get all", "test")
+        elif cmd['prefix'] == "osd smart dump-all":
+            return self.handle_osd_smart_dump_all()
 
-        elif cmd['prefix'] == "pool test":
-            return (0, "pool", "test")
+        elif cmd['prefix'] == "smart warn":
+            # TODO check why it's not in 'ceph health' output
+            self.set_health_checks({
+                'MGR_TEST_WARNING': {
+                    'severity': 'warning',
+                    'summary': 'test warning',
+                    'detail': ['details about the warning']
+                }
+            })
+            return (0, "warn", "test")
 
         else:
             # mgr should respect our self.COMMANDS and not call us for