From a542bd8e929c3213b12a31998331987a23cc574c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Tue, 6 Aug 2019 19:54:57 +0200 Subject: [PATCH 01/11] Total Chart untested --- README.md | 3 +- static/js/main.js | 161 ++++++++++++++++++++++++++++++++++++++++++++-- util/database.py | 92 +++++++++++++++++++------- 3 files changed, 224 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index c634e6c..8be7da2 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,13 @@ Inverters that do not support Speedwire or Bluetooth or that are not from the ma Current features: -* Web based visualization of daily and monthly power production +* Web based visualization of daily, monthly, yearly and overall power production * Displaying multiple inverters * Translations: English, German * Mail notification service for inverter disorders and lack of production Planned features: -* Visualization of yearly power production * Visualization of power consumption * Validation of `config.yml` with [JSON Schema](https://json-schema.org/) * Configuration of _sunportal_ via web interface diff --git a/static/js/main.js b/static/js/main.js index 5396f2a..a764433 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -132,6 +132,33 @@ function initializeCanvas() { $( "#charts" ).append( htmlYearChart ); } + if ( !$( "#chart-tot" ).length ) { + + var htmlTotChart = ` +
+
+
+
`+ ((langCode == 'de') ? 'total' : 'total' ) +`
+
+
+ + Loading ... +
+
+ +
+
+ +
+ +
+ +
+
+ `; + $( "#charts" ).append( htmlTotChart ); + } + } @@ -318,7 +345,66 @@ function initializeCharts(serial) { } }); - charts = { day: new_day_chart, month: new_month_chart, year: new_year_chart }; + var ctx_tot = document.getElementById( "chart-tot" ).getContext('2d'); + var new_tot_chart = new Chart(ctx_tot, { + type: 'bar', + data: { + labels: [], + datasets: [ {} ] + }, + options: { + responsive: true, + animation: { + duration: 1000, + easing: 'easeInOutSine' + }, + maintainAspectRatio: false, + scales: { + yAxes: [{ + ticks: { + beginAtZero: true, + fontColor: 'rgba(255,255,255,1)', + fontSize: 14 + }, + gridLines: { + color: 'rgba(255,255,255,0.1)' + }, + stacked: true + }], + xAxes: [{ + display: true, + type: 'time', + time: { + unit: 'year' + }, + ticks: { + fontColor: 'rgba(255,255,255,1)' + }, + gridLines: { + color: 'rgba(255,255,255,0.1)' + }, + stacked: true + }] + }, + legend: { + labels: { + fontColor: 'rgba(255,255,255,1)' + } + }, + tooltips: { + callbacks: { + title: function(t, d) { + if (langCode == 'de') + return moment(t[0].xLabel).format('YYYY'); + else + return moment(t[0].xLabel).format('YYYY'); + } + } + } + } + }); + + charts = { day: new_day_chart, month: new_month_chart, year: new_year_chart, tot: new_tot_chart }; } var requesting = false; @@ -330,20 +416,20 @@ function loadData(day) { clearTimeout(reloadTimer); - var request_data = { - date: currentDay + var request_data = { + date: currentDay }; // $('.chart-date').each(function() { // $(this).text(''); // }); $('.inverter-yield').each(function() { - if (! $(this).has('i.fa-circle-o-notch').length) + if (! $(this).has('i.fa-circle-o-notch').length) $(this).html('Loading ...'); }); //console.log("Requesting data for "+currentDay+" ..."); - $.ajax({ + $.ajax({ type : "POST", url : "/update", data: JSON.stringify(request_data, null, '\t'), @@ -369,6 +455,7 @@ function loadData(day) { var dayChart = charts.day; var monthChart = charts.month; var yearChart = charts.year; + var totChart = charts.tot; var currentDayDate = moment(currentDay).format('YYYY-MM-DD'); var all = response.requested.all; @@ -435,6 +522,19 @@ function loadData(day) { all.year.interval.to ]; + + // update year chart + $("#chart-year-col .chart-date").text( 'total' ); + $("#chart-year-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); + + // update scale + all.tot.interval.from = moment.unix(all.tot.interval.from); + all.tot.interval.to = moment.unix(all.tot.interval.to); + totChart.data.labels = [ + all.tot.interval.from, + all.tot.interval.to + ]; + } var chart_num = 0; @@ -496,6 +596,20 @@ function loadData(day) { inv_data.year.data = tmp2; } } + + if (inv_data.tot.data.length > 0) { + // for month data with missing timestamps at the beginning + if (all.tot.data[0].time < inv_data.tot.data[0].time) { + var tmp = [], i = 0; + var first_ts = inv_data.tot.data[0].time; + while (all.tot.data[i].time < first_ts) { + tmp.push( { 'time': all.tot.data[i].time, 'power': 0 } ); + i++; + } + var tmp2 = tmp.concat(inv_data.tot.data); + inv_data.tot.data = tmp2; + } + } // WORKAROUND FOR CHART.JS BUG END @@ -565,6 +679,31 @@ function loadData(day) { } + // update tot chart + inv_data.tot.data.forEach(function(obj) { + time = moment.unix(obj.time).startOf('year'); + + if (!moment(time).isSame(moment(), 'year')) { + obj.x = time; + obj.y = obj.power / 1000; + } + delete obj.time; + delete obj.power; + }) + + var is_label_not_defined = (totChart.data.datasets[chart_num] && (totChart.data.datasets[chart_num].label != response.today.inverters[k].name)) + if ( is_label_not_defined || !totChart.data.datasets[chart_num]) { + totChart.data.datasets[chart_num] = { + label: response.today.inverters[k].name, + data: inv_data.tot.data, + borderWidth: 1, + pointRadius: 0 + } + } else { + totChart.data.datasets[chart_num].data = inv_data.tot.data; + } + + chart_num++; } } @@ -596,6 +735,14 @@ function loadData(day) { } yearChart.update(); + totChart.data.datasets.sort(sort_inverter_datasets_alphabetically); // sort alphabetically + for (var i=0; i Date: Sun, 11 Aug 2019 12:48:16 +0200 Subject: [PATCH 02/11] Huge update --- static/js/main.js | 97 +++++++++++++++++++++++++++++++------------- templates/index.html | 14 +++---- util/database.py | 57 +++++++++++++++++--------- 3 files changed, 114 insertions(+), 54 deletions(-) diff --git a/static/js/main.js b/static/js/main.js index a764433..2006efe 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -57,11 +57,11 @@ function initializeCanvas() {
-
`+ ((langCode == 'de') ? 'tag' : 'day' ) +`
+
`+ ((langCode == 'de') ? 'Tag' : 'Day' ) +`
- loading ... + Loading ...
@@ -83,7 +83,7 @@ function initializeCanvas() {
-
`+ ((langCode == 'de') ? 'monat' : 'month' ) +`
+
`+ ((langCode == 'de') ? 'Monat' : 'Month' ) +`
@@ -111,7 +111,7 @@ function initializeCanvas() {
-
`+ ((langCode == 'de') ? 'jahr' : 'year' ) +`
+
`+ ((langCode == 'de') ? 'Jahr' : 'Year' ) +`
@@ -135,10 +135,10 @@ function initializeCanvas() { if ( !$( "#chart-tot" ).length ) { var htmlTotChart = ` -
+
-
`+ ((langCode == 'de') ? 'total' : 'total' ) +`
+
`+ ((langCode == 'de') ? 'Total' : 'Total' ) +`
@@ -215,12 +215,18 @@ function initializeCharts(serial) { }, tooltips: { callbacks: { - title: function(t, d) { + title: function(e, legendItem) { + var dateArray = e[0].xLabel.split(" "); + var day = dateArray[1].split(",")[0]; + var clockArray = dateArray[3].split(":"); + if (clockArray[0] == "12" && dateArray[4] == "am") { + clockArray[0] = 0; + } + if (clockArray[0] != "12" && dateArray[4] == "pm") { + clockArray[0] = parseInt(clockArray[0]) + 12; + } if (langCode == 'de') - return moment(t[0].xLabel).format('HH:mm [Uhr]'); - else - return moment(t[0].xLabel).format('hh:mm A'); - + return day + ". " + dateArray[0] + " " + dateArray[2] + ", " + clockArray[0] + ":" + clockArray[1] + " Uhr"; } } } @@ -275,13 +281,23 @@ function initializeCharts(serial) { }, tooltips: { callbacks: { - title: function(t, d) { + title: function(e, legendItem) { + var index = e[0].index; + var thisDay = legendItem.datasets[0].data[index].x._i; if (langCode == 'de') - return moment(t[0].xLabel).format('DD.MM.YYYY'); + return moment(thisDay).format('DD.MM.YYYY'); else - return moment(t[0].xLabel).format('YYYY/MM/DD'); + return moment(thisDay).format('YYYY/MM/DD'); } } + }, + onClick: function(e, legendItem) { + var index = legendItem[0]._index; + var newDay = legendItem[0]._chart.data.datasets[0].data[index].x._i; + newDay = moment(newDay).format('YYYY-MM-DD'); + currentDay = newDay; + + loadData(); } } }); @@ -334,13 +350,22 @@ function initializeCharts(serial) { }, tooltips: { callbacks: { - title: function(t, d) { + title: function(e, legendItem) { + var index = e[0].index; + var month = legendItem.datasets[0].data[index].x._i; if (langCode == 'de') - return moment(t[0].xLabel).format('MMMM'); + return moment(month).format('MMMM YYYY'); else - return moment(t[0].xLabel).format('MMMM'); + return moment(month).format('YYYY MMMM'); } } + }, + onClick: function(e, legendItem) { + var index = legendItem[0]._index; + var month = legendItem[0]._chart.data.datasets[0].data[index].x._i; + currentDay = checkDateValid(moment(month).format('YYYY-MM') + "-" + currentDay.split("-")[2]); + + loadData(); } } }); @@ -393,13 +418,18 @@ function initializeCharts(serial) { }, tooltips: { callbacks: { - title: function(t, d) { - if (langCode == 'de') - return moment(t[0].xLabel).format('YYYY'); - else - return moment(t[0].xLabel).format('YYYY'); + title: function(e, legendItem) { + var index = e[0].index; + var year = legendItem.datasets[0].data[index].x._i; + return moment(year).format('YYYY'); } } + }, + onClick: function(e, legendItem) { + var index = legendItem[0]._index; + var year = legendItem[0]._chart.data.datasets[0].data[index].x._i; + currentDay = checkDateValid(moment(year).format('YYYY') + "-" + currentDay.split("-")[1] + "-" + currentDay.split("-")[2]); + loadData(); } } }); @@ -524,8 +554,8 @@ function loadData(day) { // update year chart - $("#chart-year-col .chart-date").text( 'total' ); - $("#chart-year-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); + $("#chart-tot-col .chart-date").text( 'total' ); + $("#chart-tot-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); // update scale all.tot.interval.from = moment.unix(all.tot.interval.from); @@ -658,10 +688,10 @@ function loadData(day) { inv_data.year.data.forEach(function(obj) { time = moment.unix(obj.time).startOf('month'); - if (!moment(time).isSame(moment(), 'month')) { + //if (!moment(time).isSame(moment(), 'month')) { obj.x = time; obj.y = obj.power / 1000; - } + //} delete obj.time; delete obj.power; }) @@ -683,10 +713,10 @@ function loadData(day) { inv_data.tot.data.forEach(function(obj) { time = moment.unix(obj.time).startOf('year'); - if (!moment(time).isSame(moment(), 'year')) { + //if (!moment(time).isSame(moment(), 'year')) { obj.x = time; obj.y = obj.power / 1000; - } + //} delete obj.time; delete obj.power; }) @@ -881,6 +911,17 @@ function getColorShades(num, color) { return shades; } +function checkDateValid(date){ + var d = new Date(date); + var day = date.split("-")[2]; + while (isNaN(d)) { + day--; + date = date.split("-")[0] + "-" + date.split("-")[1] + "-" + day; + d = new Date(date); + } + return date; +} + const shadeBlendConvert = function (p, from, to) { if(typeof(p)!="number"||p<-1||p>1||typeof(from)!="string"||(from[0]!='r'&&from[0]!='#')||(to&&typeof(to)!="string"))return null; //ErrorCheck if(!this.sbcRip)this.sbcRip=(d)=>{ diff --git a/templates/index.html b/templates/index.html index f5e2bb6..0269a70 100644 --- a/templates/index.html +++ b/templates/index.html @@ -21,13 +21,13 @@
-

sunportal

+

Sunportal

-
today's yield
-
heutiger ertrag
+
Today's yield
+
Heutiger Ertrag

loading ... @@ -37,8 +37,8 @@

heutiger ertrag
-
total yield
-
gesamtertrag
+
Total yield
+
Gesamtertrag

Loading ... @@ -49,10 +49,10 @@

gesamtertrag
CO2 savings
-
CO2-einsparung
+
CO2-Einsparung

- loading ... + Loading ...

diff --git a/util/database.py b/util/database.py index 5634995..f61aeb0 100644 --- a/util/database.py +++ b/util/database.py @@ -17,6 +17,12 @@ def __init__(self, config): self.local_timezone = self.get_local_timezone() def get(self, date): + tot_start, tot_end = self.get_epoch_tot() + int_date = int(datetime(int(date.split('-')[0]), int(date.split('-')[1]), int(date.split('-')[2]), 00, 00, 00, tzinfo=pytz.utc).timestamp()) + if int_date < tot_start: + date = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d') + if int_date > tot_end: + date = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d') data = dict() data['today'] = self.get_today() data['requested'] = self.get_requested(date) @@ -117,7 +123,7 @@ def get_requested_day(self, date): row = self.c.fetchone() if row and row[0]: data['total'] = row[0] - else: data['total'] = 0 + else: data['total'] = self.get_today()['dayTotal'] query = ''' @@ -169,10 +175,8 @@ def get_requested_day_for_inverter(self, inverter_serial, date): self.c.execute(query, (day_start, day_end, inverter_serial)) res = self.c.fetchone() - if res and res[0]: - data['total'] = res[0] - else: - data['total'] = 0 + if res and res[0]: data['total'] = res[0] + else: data['total'] = self.get_today()['inverters'][inverter_serial]['dayTotal'] query = ''' SELECT MIN(TimeStamp) as Min, MAX(TimeStamp) as Max @@ -288,7 +292,8 @@ def get_requested_year(self, date): self.c.execute(query, (current_month_start_local, current_month_end_local)) - month_total = self.c.fetchone()[0] + gen_date = datetime(int(date.split('-')[0]), i, 1) + month_total = self.get_requested_month(str(gen_date).split(' ')[0])['total'] if month_total is None: month_total = 0 @@ -342,7 +347,8 @@ def get_requested_year_for_inverter(self, inverter_serial, date): self.c.execute(query, (current_month_start_local, current_month_end_local, inverter_serial)) - month_total = self.c.fetchone()[0] + gen_date = datetime(int(date.split('-')[0]), i, 1) + month_total = self.get_requested_month_for_inverter(inverter_serial, str(gen_date).split(' ')[0])['total'] if month_total is None: month_total = 0 @@ -376,15 +382,16 @@ def get_requested_tot(self): tot_start, tot_end = self.get_epoch_tot() data['interval'] = {'from': self.convert_local_ts_to_utc(tot_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(tot_end, self.local_timezone)} - string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d %H:%M:%S') - string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d %H:%M:%S') - - for i in range(int(string_start.split('-')[0]), int(string_end.split('-')[0])): - gen_date = i + "-01-01 00:00:00" - data['data'].append({'time': gen_date, 'power': get_requested_year(gen_date)['total']}) + string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d') + string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d') data['data'] = list() + for i in range(int(string_start.split('-')[0]), int(string_end.split('-')[0]) + 1): + gen_date = str(i) + "-01-01" + gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + data['data'].append({'time': gen_ts, 'power': self.get_requested_year(gen_date)['total']}) + return data def get_requested_tot_for_inverter(self, inverter_serial): @@ -393,15 +400,16 @@ def get_requested_tot_for_inverter(self, inverter_serial): tot_start, tot_end = self.get_epoch_tot() data['interval'] = {'from': self.convert_local_ts_to_utc(tot_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(tot_end, self.local_timezone)} - string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d %H:%M:%S') - string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d %H:%M:%S') - - for i in range(int(string_start.split('-')[0]), int(string_end.split('-')[0])): - gen_date = i + "-01-01 00:00:00" - data['data'].append({'time': gen_date, 'power': get_requested_year(inverter_serial, gen_date)['total']}) + string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d') + string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d') data['data'] = list() + for i in range(int(string_start.split('-')[0]), int(string_end.split('-')[0]) + 1): + gen_date = str(i) + "-01-01" + gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + data['data'].append({'time': gen_ts, 'power': self.get_requested_year_for_inverter(inverter_serial, gen_date)['total']}) + return data def get_inverters(self): @@ -464,6 +472,17 @@ def get_epoch_tot(self): self.c.execute(query) epoch_start, epoch_end = self.c.fetchone() + + query = ''' + SELECT MIN(TimeStamp) as Min, MAX(TimeStamp) as Max + FROM ( SELECT TimeStamp FROM DayData GROUP BY TimeStamp ); + ''' + + self.c.execute(query) + day_start, day_end = self.c.fetchone() + if day_end > epoch_end: + epoch_end = day_end + return epoch_start, epoch_end def get_epoch_month_ends_for_year(self, date): From 32a3a8d7419b51fd70b03248700d06eada7d394a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Sun, 11 Aug 2019 20:22:16 +0200 Subject: [PATCH 03/11] Current Day in Month --- util/database.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/util/database.py b/util/database.py index f61aeb0..3f06998 100644 --- a/util/database.py +++ b/util/database.py @@ -214,6 +214,11 @@ def get_requested_month(self, date): data['data'].append({'time': self.convert_local_ts_to_utc(row[0], self.local_timezone), 'power': row[1]}) month_total += row[1] + tot_start, tot_end = self.get_epoch_tot() + if date.split('-')[0] + '-' + date.split('-')[1] == datetime.utcfromtimestamp(tot_end).strftime('%Y-%m'): + data['data'].append({'time': self.convert_local_ts_to_utc(tot_end, self.local_timezone), 'power': self.get_today()['dayTotal']}) + month_total += self.get_today()['dayTotal'] + data['total'] = month_total query = ''' @@ -249,6 +254,11 @@ def get_requested_month_for_inverter(self, inverter_serial, date): data['data'].append({'time': self.convert_local_ts_to_utc(row[0], self.local_timezone), 'power': row[1]}) month_total += row[1] + tot_start, tot_end = self.get_epoch_tot() + if date.split('-')[0] + '-' + date.split('-')[1] == datetime.utcfromtimestamp(tot_end).strftime('%Y-%m'): + data['data'].append({'time': self.convert_local_ts_to_utc(tot_end, self.local_timezone), 'power': self.get_today()['inverters'][inverter_serial]['dayTotal']}) + month_total += self.get_today()['inverters'][inverter_serial]['dayTotal'] + data['total'] = month_total query = ''' From 83e7daef7ff3533189e1e0c9d5c707040d52a7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Mon, 12 Aug 2019 12:37:27 +0200 Subject: [PATCH 04/11] Optimize load times 1 --- static/js/main.js | 2 +- util/database.py | 62 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/static/js/main.js b/static/js/main.js index 2006efe..8b88063 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -554,7 +554,7 @@ function loadData(day) { // update year chart - $("#chart-tot-col .chart-date").text( 'total' ); + $("#chart-tot-col .chart-date").text( 'Total' ); $("#chart-tot-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); // update scale diff --git a/util/database.py b/util/database.py index 3f06998..d36b34e 100644 --- a/util/database.py +++ b/util/database.py @@ -7,6 +7,10 @@ class Database(): + hasRun = False + data = 0 + inverters = 0 + year_data = dict() def __init__(self, config): self.config = config @@ -15,6 +19,8 @@ def __init__(self, config): self.c = self.db.cursor() self.local_timezone = self.get_local_timezone() + global hasRun + hasRun = False def get(self, date): tot_start, tot_end = self.get_epoch_tot() @@ -64,26 +70,40 @@ def get_today(self): return data def get_requested(self, date): + global hasRun + global data + global inverters + global year_data - data = dict() + if hasRun is False: + data = dict() data['date'] = date - data['all'] = dict() + if hasRun is False: + data['all'] = dict() data['all']['day'] = self.get_requested_day(date) data['all']['month'] = self.get_requested_month(date) - data['all']['year'] = self.get_requested_year(date) - data['all']['tot'] = self.get_requested_tot() + if hasRun is False: + data['all']['year'] = self.get_requested_year(date) + data['all']['tot'] = self.get_requested_tot() - data['inverters'] = dict() + data['inverters'] = dict() - inverters = self.get_inverters() + inverters = self.get_inverters() + else: + data['all']['year'] = year_data[date.split('-')[0]] for inv in inverters: - data['inverters'][inv['serial']] = { 'day': [], 'month': [] } + if hasRun is False: + data['inverters'][inv['serial']] = { 'day': [], 'month': [], 'year': [], 'tot': [] } data['inverters'][inv['serial']]['day'] = self.get_requested_day_for_inverter(inv['serial'], date) data['inverters'][inv['serial']]['month'] = self.get_requested_month_for_inverter(inv['serial'], date) - data['inverters'][inv['serial']]['year'] = self.get_requested_year_for_inverter(inv['serial'], date) - data['inverters'][inv['serial']]['tot'] = self.get_requested_tot_for_inverter(inv['serial']) + if hasRun is False: + data['inverters'][inv['serial']]['year'] = self.get_requested_year_for_inverter(inv['serial'], date) + data['inverters'][inv['serial']]['tot'] = self.get_requested_tot_for_inverter(inv['serial']) + else: + data['inverters'][inv['serial']]['year'] = year_data[inv['serial']][date.split('-')[0]] + hasRun = True return data @@ -387,6 +407,10 @@ def get_requested_year_for_inverter(self, inverter_serial, date): return data def get_requested_tot(self): + global year_data + if hasRun is False: + year_data = dict() + data = dict() tot_start, tot_end = self.get_epoch_tot() @@ -400,25 +424,33 @@ def get_requested_tot(self): for i in range(int(string_start.split('-')[0]), int(string_end.split('-')[0]) + 1): gen_date = str(i) + "-01-01" gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) - data['data'].append({'time': gen_ts, 'power': self.get_requested_year(gen_date)['total']}) + year = self.get_requested_year(gen_date) + data['data'].append({'time': gen_ts, 'power': year['total']}) + year_data[str(i)] = year return data def get_requested_tot_for_inverter(self, inverter_serial): + global year_data + if hasRun is False: + year_data[inverter_serial] = dict() + data = dict() tot_start, tot_end = self.get_epoch_tot() data['interval'] = {'from': self.convert_local_ts_to_utc(tot_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(tot_end, self.local_timezone)} - string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d') - string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d') + string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y') + string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y') data['data'] = list() - for i in range(int(string_start.split('-')[0]), int(string_end.split('-')[0]) + 1): + for i in range(int(string_start), int(string_end) + 1): gen_date = str(i) + "-01-01" gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) - data['data'].append({'time': gen_ts, 'power': self.get_requested_year_for_inverter(inverter_serial, gen_date)['total']}) + year = self.get_requested_year_for_inverter(inverter_serial, gen_date) + data['data'].append({'time': gen_ts, 'power': year['total']}) + year_data[inverter_serial][str(i)] = year return data @@ -475,6 +507,8 @@ def get_epoch_year(self, date): return epoch_start, epoch_end def get_epoch_tot(self): + global hasRun + query = ''' SELECT MIN(TimeStamp) as Min, MAX(TimeStamp) as Max FROM ( SELECT TimeStamp FROM MonthData GROUP BY TimeStamp ); From e2eb48a4db3059e1ae1f4a386f26c74e3d6a2ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Tue, 13 Aug 2019 13:15:52 +0200 Subject: [PATCH 05/11] Fix some strings --- static/js/main.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/static/js/main.js b/static/js/main.js index 8b88063..359b83b 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -553,7 +553,7 @@ function loadData(day) { ]; - // update year chart + // update tot chart $("#chart-tot-col .chart-date").text( 'Total' ); $("#chart-tot-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); @@ -882,9 +882,8 @@ function getDayStringForPrint() { function getMonthStringForPrint() { var date = moment(currentDay).format('YYYY-MM-DD'); - //console.log('debug', moment(date).locale(langCode)); if (currentDay) { - return moment(date).locale(langCode).format('MMMM YYYY').toLowerCase(); + return moment(date).locale(langCode).format('MMMM YYYY'); } else { return ''; } @@ -893,9 +892,8 @@ function getMonthStringForPrint() { function getYearStringForPrint() { var date = moment(currentDay).format('YYYY-MM-DD'); - //console.log('debug', moment(date).locale(langCode)); if (currentDay) { - return moment(date).locale(langCode).format('YYYY').toLowerCase(); + return moment(date).locale(langCode).format('YYYY'); } else { return ''; } From 8dca723c5a9282391fa80075ca79e6321aa3ce57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Tue, 13 Aug 2019 19:38:04 +0200 Subject: [PATCH 06/11] Moved selective loading to frontend --- static/js/main.js | 445 +++++++++++++++++++++++++--------------------- sunportal.py | 15 +- util/database.py | 59 +++--- 3 files changed, 271 insertions(+), 248 deletions(-) diff --git a/static/js/main.js b/static/js/main.js index 359b83b..53af27c 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -28,7 +28,7 @@ $(document).ready(function () { $('#content').fadeIn(1000, function() { /* ----- load data from database via ajax ------ */ - loadData(); + loadData(3); }); @@ -297,7 +297,7 @@ function initializeCharts(serial) { newDay = moment(newDay).format('YYYY-MM-DD'); currentDay = newDay; - loadData(); + loadData(0); } } }); @@ -365,7 +365,7 @@ function initializeCharts(serial) { var month = legendItem[0]._chart.data.datasets[0].data[index].x._i; currentDay = checkDateValid(moment(month).format('YYYY-MM') + "-" + currentDay.split("-")[2]); - loadData(); + loadData(1); } } }); @@ -429,7 +429,7 @@ function initializeCharts(serial) { var index = legendItem[0]._index; var year = legendItem[0]._chart.data.datasets[0].data[index].x._i; currentDay = checkDateValid(moment(year).format('YYYY') + "-" + currentDay.split("-")[1] + "-" + currentDay.split("-")[2]); - loadData(); + loadData(2); } } }); @@ -438,7 +438,7 @@ function initializeCharts(serial) { } var requesting = false; -function loadData(day) { +function loadData(load_mode = 1) { // only one request at a time if (requesting) return; @@ -447,15 +447,23 @@ function loadData(day) { clearTimeout(reloadTimer); var request_data = { - date: currentDay + date: currentDay, + requested_data: { + day: load_mode >= 0, + month: load_mode >= 1, + year: load_mode >= 2, + tot: load_mode >= 3 + } }; // $('.chart-date').each(function() { // $(this).text(''); // }); + var inv_count = 0; $('.inverter-yield').each(function() { - if (! $(this).has('i.fa-circle-o-notch').length) + if (inv_count <= load_mode && ! $(this).has('i.fa-circle-o-notch').length) $(this).html('Loading ...'); + inv_count++; }); //console.log("Requesting data for "+currentDay+" ..."); @@ -472,7 +480,7 @@ function loadData(day) { }, success: function(response) { - console.log(response) + //console.log(response) // current information $("#dayTotal").text( addPrefix(response.today.dayTotal) + "Wh" ); @@ -491,79 +499,85 @@ function loadData(day) { var all = response.requested.all; { - // update day chart - $("#chart-day-col .chart-date").text(getDayStringForPrint()); - $("#chart-day-col .inverter-yield").text( addPrefix(all.day.total) + "Wh"); - - // show or hide navigation arrows on day chart - $("#chart-day-col .navigation-right").each(function() { - $(this).css("visibility", (all.day.hasNext ? "visible" : "hidden")); - }); - $("#chart-day-col .navigation-left").each(function() { - $(this).css("visibility", (all.day.hasPrevious ? "visible" : "hidden")); - }); - - // update scale - all.day.interval.from = moment.unix(all.day.interval.from); - all.day.interval.to= moment.unix(all.day.interval.to); - dayChart.data.labels = [ - all.day.interval.from, - all.day.interval.to - ]; - - // update month chart - $("#chart-month-col .chart-date").text( getMonthStringForPrint() ); - $("#chart-month-col .inverter-yield").text( addPrefix(all.month.total) + "Wh"); - - // show or hide navigation arrows on day chart - $("#chart-month-col .navigation-right").each(function() { - $(this).css("visibility", (all.month.hasNext ? "visible" : "hidden")); - }); - $("#chart-month-col .navigation-left").each(function() { - $(this).css("visibility", (all.month.hasPrevious ? "visible" : "hidden")); - }); - - // update scale - all.month.interval.from = moment.unix(all.month.interval.from); - all.month.interval.to = moment.unix(all.month.interval.to); - monthChart.data.labels = [ - all.month.interval.from, - all.month.interval.to - ]; - - - // update year chart - $("#chart-year-col .chart-date").text( getYearStringForPrint() ); - $("#chart-year-col .inverter-yield").text( addPrefix(all.year.total) + "Wh"); - - // show or hide navigation arrows on day chart - $("#chart-year-col .navigation-right").each(function() { - $(this).css("visibility", (all.year.hasNext ? "visible" : "hidden")); - }); - $("#chart-year-col .navigation-left").each(function() { - $(this).css("visibility", (all.year.hasPrevious ? "visible" : "hidden")); - }); - - // update scale - all.year.interval.from = moment.unix(all.year.interval.from).subtract(30.4, 'days'); // data has first day of month as data point - all.year.interval.to = moment.unix(all.year.interval.to); - yearChart.data.labels = [ - all.year.interval.from, - all.year.interval.to - ]; - - - // update tot chart - $("#chart-tot-col .chart-date").text( 'Total' ); - $("#chart-tot-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); - - // update scale - all.tot.interval.from = moment.unix(all.tot.interval.from); - all.tot.interval.to = moment.unix(all.tot.interval.to); - totChart.data.labels = [ - all.tot.interval.from, - all.tot.interval.to - ]; + if (load_mode >= 0) { + // update day chart + $("#chart-day-col .chart-date").text(getDayStringForPrint()); + $("#chart-day-col .inverter-yield").text( addPrefix(all.day.total) + "Wh"); + + // show or hide navigation arrows on day chart + $("#chart-day-col .navigation-right").each(function() { + $(this).css("visibility", (all.day.hasNext ? "visible" : "hidden")); + }); + $("#chart-day-col .navigation-left").each(function() { + $(this).css("visibility", (all.day.hasPrevious ? "visible" : "hidden")); + }); + + // update scale + all.day.interval.from = moment.unix(all.day.interval.from); + all.day.interval.to= moment.unix(all.day.interval.to); + dayChart.data.labels = [ + all.day.interval.from, + all.day.interval.to + ]; + } + + if (load_mode >= 1) { + // update month chart + $("#chart-month-col .chart-date").text( getMonthStringForPrint() ); + $("#chart-month-col .inverter-yield").text( addPrefix(all.month.total) + "Wh"); + + // show or hide navigation arrows on day chart + $("#chart-month-col .navigation-right").each(function() { + $(this).css("visibility", (all.month.hasNext ? "visible" : "hidden")); + }); + $("#chart-month-col .navigation-left").each(function() { + $(this).css("visibility", (all.month.hasPrevious ? "visible" : "hidden")); + }); + + // update scale + all.month.interval.from = moment.unix(all.month.interval.from); + all.month.interval.to = moment.unix(all.month.interval.to); + monthChart.data.labels = [ + all.month.interval.from, + all.month.interval.to + ]; + } + + if (load_mode >= 2) { + // update year chart + $("#chart-year-col .chart-date").text( getYearStringForPrint() ); + $("#chart-year-col .inverter-yield").text( addPrefix(all.year.total) + "Wh"); + + // show or hide navigation arrows on day chart + $("#chart-year-col .navigation-right").each(function() { + $(this).css("visibility", (all.year.hasNext ? "visible" : "hidden")); + }); + $("#chart-year-col .navigation-left").each(function() { + $(this).css("visibility", (all.year.hasPrevious ? "visible" : "hidden")); + }); + + // update scale + all.year.interval.from = moment.unix(all.year.interval.from).subtract(30.4, 'days'); // data has first day of month as data point + all.year.interval.to = moment.unix(all.year.interval.to); + yearChart.data.labels = [ + all.year.interval.from, + all.year.interval.to + ]; + } + + if (load_mode >= 3) { + // update tot chart + $("#chart-tot-col .chart-date").text( 'Total' ); + $("#chart-tot-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); + + // update scale + all.tot.interval.from = moment.unix(all.tot.interval.from); + all.tot.interval.to = moment.unix(all.tot.interval.to); + totChart.data.labels = [ + all.tot.interval.from, + all.tot.interval.to + ]; + } } @@ -575,7 +589,7 @@ function loadData(day) { // WORKAROUND FOR CHART.JS STACKED CHARTS BUG // ISSUE: https://github.com/chartjs/Chart.js/issues/5484 - if (inv_data.day.data.length > 0) { + if (load_mode >= 0 && inv_data.day.data.length > 0) { // for day data with missing timestamps at the beginning if (all.day.data[0].time < inv_data.day.data[0].time) { @@ -599,7 +613,7 @@ function loadData(day) { } } - if (inv_data.month.data.length > 0) { + if (load_mode >= 1 && inv_data.month.data.length > 0) { // for month data with missing timestamps at the beginning if (all.month.data[0].time < inv_data.month.data[0].time) { var tmp = [], i = 0; @@ -613,7 +627,7 @@ function loadData(day) { } } - if (inv_data.year.data.length > 0) { + if (load_mode >= 2 && inv_data.year.data.length > 0) { // for month data with missing timestamps at the beginning if (all.year.data[0].time < inv_data.year.data[0].time) { var tmp = [], i = 0; @@ -627,7 +641,7 @@ function loadData(day) { } } - if (inv_data.tot.data.length > 0) { + if (load_mode >= 3 && inv_data.tot.data.length > 0) { // for month data with missing timestamps at the beginning if (all.tot.data[0].time < inv_data.tot.data[0].time) { var tmp = [], i = 0; @@ -643,95 +657,100 @@ function loadData(day) { // WORKAROUND FOR CHART.JS BUG END - // update day chart - inv_data.day.data.forEach(function(obj) { - obj.x = moment.unix(obj.time); - obj.y = obj.power / 1000; - delete obj.time; - delete obj.power; - }) - var is_label_not_defined = (dayChart.data.datasets[chart_num] && (dayChart.data.datasets[chart_num].label != response.today.inverters[k].name)) - if (is_label_not_defined || !dayChart.data.datasets[chart_num]) { - dayChart.data.datasets[chart_num] = { - label: response.today.inverters[k].name, - data: inv_data.day.data, - borderWidth: 1, - pointRadius: 0 - } - } else { - dayChart.data.datasets[chart_num].data = inv_data.day.data; - } - - - // update month chart - inv_data.month.data.forEach(function(obj) { - obj.x = moment.unix(obj.time).startOf('day').add(12, 'h'); - obj.y = obj.power / 1000; - delete obj.time; - delete obj.power; - }) - - var is_label_not_defined = (monthChart.data.datasets[chart_num] && (monthChart.data.datasets[chart_num].label != response.today.inverters[k].name)) - if ( is_label_not_defined || !monthChart.data.datasets[chart_num]) { - monthChart.data.datasets[chart_num] = { - label: response.today.inverters[k].name, - data: inv_data.month.data, - borderWidth: 1, - pointRadius: 0 - } - } else { - monthChart.data.datasets[chart_num].data = inv_data.month.data; - } - + if (load_mode >= 0) { + // update day chart + inv_data.day.data.forEach(function(obj) { + obj.x = moment.unix(obj.time); + obj.y = obj.power / 1000; + delete obj.time; + delete obj.power; + }) + var is_label_not_defined = (dayChart.data.datasets[chart_num] && (dayChart.data.datasets[chart_num].label != response.today.inverters[k].name)) + if (is_label_not_defined || !dayChart.data.datasets[chart_num]) { + dayChart.data.datasets[chart_num] = { + label: response.today.inverters[k].name, + data: inv_data.day.data, + borderWidth: 1, + pointRadius: 0 + } + } else { + dayChart.data.datasets[chart_num].data = inv_data.day.data; + } + } - // update year chart - inv_data.year.data.forEach(function(obj) { - time = moment.unix(obj.time).startOf('month'); - - //if (!moment(time).isSame(moment(), 'month')) { - obj.x = time; - obj.y = obj.power / 1000; - //} - delete obj.time; - delete obj.power; - }) - - var is_label_not_defined = (yearChart.data.datasets[chart_num] && (yearChart.data.datasets[chart_num].label != response.today.inverters[k].name)) - if ( is_label_not_defined || !yearChart.data.datasets[chart_num]) { - yearChart.data.datasets[chart_num] = { - label: response.today.inverters[k].name, - data: inv_data.year.data, - borderWidth: 1, - pointRadius: 0 - } - } else { - yearChart.data.datasets[chart_num].data = inv_data.year.data; - } + if (load_mode >= 1) { + // update month chart + inv_data.month.data.forEach(function(obj) { + obj.x = moment.unix(obj.time).startOf('day').add(12, 'h'); + obj.y = obj.power / 1000; + delete obj.time; + delete obj.power; + }) + + var is_label_not_defined = (monthChart.data.datasets[chart_num] && (monthChart.data.datasets[chart_num].label != response.today.inverters[k].name)) + if ( is_label_not_defined || !monthChart.data.datasets[chart_num]) { + monthChart.data.datasets[chart_num] = { + label: response.today.inverters[k].name, + data: inv_data.month.data, + borderWidth: 1, + pointRadius: 0 + } + } else { + monthChart.data.datasets[chart_num].data = inv_data.month.data; + } + } + if (load_mode >= 2) { + // update year chart + inv_data.year.data.forEach(function(obj) { + time = moment.unix(obj.time).startOf('month'); + + //if (!moment(time).isSame(moment(), 'month')) { + obj.x = time; + obj.y = obj.power / 1000; + //} + delete obj.time; + delete obj.power; + }) + + var is_label_not_defined = (yearChart.data.datasets[chart_num] && (yearChart.data.datasets[chart_num].label != response.today.inverters[k].name)) + if ( is_label_not_defined || !yearChart.data.datasets[chart_num]) { + yearChart.data.datasets[chart_num] = { + label: response.today.inverters[k].name, + data: inv_data.year.data, + borderWidth: 1, + pointRadius: 0 + } + } else { + yearChart.data.datasets[chart_num].data = inv_data.year.data; + } + } - // update tot chart - inv_data.tot.data.forEach(function(obj) { - time = moment.unix(obj.time).startOf('year'); - - //if (!moment(time).isSame(moment(), 'year')) { - obj.x = time; - obj.y = obj.power / 1000; - //} - delete obj.time; - delete obj.power; - }) - - var is_label_not_defined = (totChart.data.datasets[chart_num] && (totChart.data.datasets[chart_num].label != response.today.inverters[k].name)) - if ( is_label_not_defined || !totChart.data.datasets[chart_num]) { - totChart.data.datasets[chart_num] = { - label: response.today.inverters[k].name, - data: inv_data.tot.data, - borderWidth: 1, - pointRadius: 0 - } - } else { - totChart.data.datasets[chart_num].data = inv_data.tot.data; - } + if (load_mode >= 3) { + // update tot chart + inv_data.tot.data.forEach(function(obj) { + time = moment.unix(obj.time).startOf('year'); + + //if (!moment(time).isSame(moment(), 'year')) { + obj.x = time; + obj.y = obj.power / 1000; + //} + delete obj.time; + delete obj.power; + }) + + var is_label_not_defined = (totChart.data.datasets[chart_num] && (totChart.data.datasets[chart_num].label != response.today.inverters[k].name)) + if ( is_label_not_defined || !totChart.data.datasets[chart_num]) { + totChart.data.datasets[chart_num] = { + label: response.today.inverters[k].name, + data: inv_data.tot.data, + borderWidth: 1, + pointRadius: 0 + } + } else { + totChart.data.datasets[chart_num].data = inv_data.tot.data; + } + } chart_num++; @@ -741,37 +760,45 @@ function loadData(day) { var backgroundColorShades = getColorShades(dayChart.data.datasets.length, 'rgba(227,6,19,0.3)') var borderColorShades = getColorShades(dayChart.data.datasets.length, 'rgba(227,6,19,1)') - dayChart.data.datasets.sort(sort_inverter_datasets_alphabetically); // sort alphabetically - for (var i=0; i= 0) { + dayChart.data.datasets.sort(sort_inverter_datasets_alphabetically); // sort alphabetically + for (var i=0; i= 1) { + monthChart.data.datasets.sort(sort_inverter_datasets_alphabetically); // sort alphabetically + for (var i=0; i= 2) { + yearChart.data.datasets.sort(sort_inverter_datasets_alphabetically); // sort alphabetically + for (var i=0; i= 3) { + totChart.data.datasets.sort(sort_inverter_datasets_alphabetically); // sort alphabetically + for (var i=0; i Date: Tue, 13 Aug 2019 20:00:06 +0200 Subject: [PATCH 07/11] Some unifiying and workaround tot chart --- util/database.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/database.py b/util/database.py index cc65283..4d5af10 100644 --- a/util/database.py +++ b/util/database.py @@ -405,12 +405,12 @@ def get_requested_tot(self): tot_start, tot_end = self.get_epoch_tot() data['interval'] = {'from': self.convert_local_ts_to_utc(tot_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(tot_end, self.local_timezone)} - string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d') - string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d') + string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y') + string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y') data['data'] = list() - for i in range(int(string_start.split('-')[0]), int(string_end.split('-')[0]) + 1): + for i in range(int(string_start) - 1, int(string_end) + 1): gen_date = str(i) + "-01-01" gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) year = self.get_requested_year(gen_date) @@ -429,7 +429,7 @@ def get_requested_tot_for_inverter(self, inverter_serial): data['data'] = list() - for i in range(int(string_start), int(string_end) + 1): + for i in range(int(string_start) - 1, int(string_end) + 1): gen_date = str(i) + "-01-01" gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) year = self.get_requested_year_for_inverter(inverter_serial, gen_date) From 7e78a3cfb076ad48485e785212529a912e27a392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Wed, 14 Aug 2019 20:20:25 +0200 Subject: [PATCH 08/11] Some fixes --- static/js/main.js | 4 ++-- util/database.py | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/static/js/main.js b/static/js/main.js index 53af27c..22e611f 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -571,8 +571,8 @@ function loadData(load_mode = 1) { $("#chart-tot-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); // update scale - all.tot.interval.from = moment.unix(all.tot.interval.from); - all.tot.interval.to = moment.unix(all.tot.interval.to); + all.tot.interval.from = moment.unix(all.tot.interval.from).subtract('years', 1); + all.tot.interval.to = moment.unix(all.tot.interval.to).add('years', 1); totChart.data.labels = [ all.tot.interval.from, all.tot.interval.to diff --git a/util/database.py b/util/database.py index 4d5af10..97df309 100644 --- a/util/database.py +++ b/util/database.py @@ -7,10 +7,6 @@ class Database(): - hasRun = False - data = 0 - inverters = 0 - year_data = dict() def __init__(self, config): self.config = config @@ -19,8 +15,6 @@ def __init__(self, config): self.c = self.db.cursor() self.local_timezone = self.get_local_timezone() - global hasRun - hasRun = False def get(self, date, requested_data): tot_start, tot_end = self.get_epoch_tot() @@ -410,7 +404,7 @@ def get_requested_tot(self): data['data'] = list() - for i in range(int(string_start) - 1, int(string_end) + 1): + for i in range(int(string_start), int(string_end) + 1): gen_date = str(i) + "-01-01" gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) year = self.get_requested_year(gen_date) @@ -429,7 +423,7 @@ def get_requested_tot_for_inverter(self, inverter_serial): data['data'] = list() - for i in range(int(string_start) - 1, int(string_end) + 1): + for i in range(int(string_start), int(string_end) + 1): gen_date = str(i) + "-01-01" gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) year = self.get_requested_year_for_inverter(inverter_serial, gen_date) From a6b54ddfc47b1196f00f886fbabed4dd8ae6fed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Wed, 14 Aug 2019 21:03:41 +0200 Subject: [PATCH 09/11] PEP8 proposal --- util/database.py | 215 +++++++++++++++++++++++++++-------------------- 1 file changed, 123 insertions(+), 92 deletions(-) diff --git a/util/database.py b/util/database.py index 97df309..bfc1d70 100644 --- a/util/database.py +++ b/util/database.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 """ """ -import pytz import sqlite3 from datetime import datetime, timedelta +import pytz class Database(): @@ -11,14 +11,15 @@ class Database(): def __init__(self, config): self.config = config self.co2_mult = self.config.get_co2_avoidance_factor() - self.db = sqlite3.connect(self.config.get_database_path(), check_same_thread=False) - self.c = self.db.cursor() + self.database = sqlite3.connect(self.config.get_database_path(), check_same_thread=False) + self.cursor = self.database.cursor() self.local_timezone = self.get_local_timezone() def get(self, date, requested_data): tot_start, tot_end = self.get_epoch_tot() - int_date = int(datetime(int(date.split('-')[0]), int(date.split('-')[1]), int(date.split('-')[2]), 00, 00, 00, tzinfo=pytz.utc).timestamp()) + date_split = date.split('-') + int_date = int(datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 00, 00, 00, tzinfo=pytz.utc).timestamp()) if int_date < tot_start: date = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d') if int_date > tot_end: @@ -53,8 +54,10 @@ def get_today(self): 'co2': inv_co2 } - if inv['etoday'] is not None: total_day += inv['etoday'] - if inv['etotal'] is not None: total += inv['etotal'] + if inv['etoday'] is not None: + total_day += inv['etoday'] + if inv['etotal'] is not None: + total += inv['etotal'] co2 += inv_co2 data['dayTotal'] = total_day @@ -81,7 +84,7 @@ def get_requested(self, date, requested_data): inverters = self.get_inverters() for inv in inverters: - data['inverters'][inv['serial']] = { 'day': [], 'month': [], 'year': [], 'tot': [] } + data['inverters'][inv['serial']] = {'day': [], 'month': [], 'year': [], 'tot': []} if requested_data['day'] is True: data['inverters'][inv['serial']]['day'] = self.get_requested_day_for_inverter(inv['serial'], date) @@ -109,8 +112,8 @@ def get_requested_day(self, date): ''' data['data'] = list() - for row in self.c.execute(query, (day_start, day_end)): - data['data'].append({ 'time': row[0], 'power': row[1] }) + for row in self.cursor.execute(query, (day_start, day_end)): + data['data'].append({'time': row[0], 'power': row[1]}) if self.get_datetime(date).date() == datetime.today().date(): @@ -118,7 +121,7 @@ def get_requested_day(self, date): SELECT SUM(EToday) as EToday FROM Inverters; ''' - self.c.execute(query) + self.cursor.execute(query) else: query = ''' SELECT SUM(DayYield) AS Power @@ -126,11 +129,13 @@ def get_requested_day(self, date): WHERE TimeStamp BETWEEN ? AND ? GROUP BY TimeStamp; ''' - self.c.execute(query, (day_start, day_end)) + self.cursor.execute(query, (day_start, day_end)) - row = self.c.fetchone() - if row and row[0]: data['total'] = row[0] - else: data['total'] = self.get_today()['dayTotal'] + row = self.cursor.fetchone() + if row and row[0]: + data['total'] = row[0] + else: + data['total'] = self.get_today()['dayTotal'] query = ''' @@ -138,14 +143,18 @@ def get_requested_day(self, date): FROM ( SELECT TimeStamp FROM DayData GROUP BY TimeStamp ); ''' - self.c.execute(query) - first_data, last_data = self.c.fetchone() + self.cursor.execute(query) + first_data, last_data = self.cursor.fetchone() - if (first_data): data['hasPrevious'] = (first_data < day_start) - else: data['hasPrevious'] = False + if first_data: + data['hasPrevious'] = (first_data < day_start) + else: + data['hasPrevious'] = False - if (last_data): data['hasNext'] = (last_data > day_end) - else: data['hasNext'] = False + if last_data: + data['hasNext'] = (last_data > day_end) + else: + data['hasNext'] = False #print(json.dumps(data, indent=4)) return data @@ -163,7 +172,7 @@ def get_requested_day_for_inverter(self, inverter_serial, date): ''' data['data'] = list() - for row in self.c.execute(query, (day_start, day_end, inverter_serial)): + for row in self.cursor.execute(query, (day_start, day_end, inverter_serial)): data['data'].append({'time': row[0], 'power': row[1]}) if self.get_datetime(date).date() == datetime.today().date(): @@ -172,32 +181,38 @@ def get_requested_day_for_inverter(self, inverter_serial, date): FROM Inverters WHERE Serial=?; ''' - self.c.execute(query, (inverter_serial,)) + self.cursor.execute(query, (inverter_serial,)) else: query = ''' SELECT DayYield AS Power FROM MonthData WHERE TimeStamp BETWEEN ? AND ? AND Serial=?; ''' - self.c.execute(query, (day_start, day_end, inverter_serial)) + self.cursor.execute(query, (day_start, day_end, inverter_serial)) - res = self.c.fetchone() - if res and res[0]: data['total'] = res[0] - else: data['total'] = self.get_today()['inverters'][inverter_serial]['dayTotal'] + res = self.cursor.fetchone() + if res and res[0]: + data['total'] = res[0] + else: + data['total'] = self.get_today()['inverters'][inverter_serial]['dayTotal'] query = ''' SELECT MIN(TimeStamp) as Min, MAX(TimeStamp) as Max FROM ( SELECT TimeStamp FROM DayData WHERE Serial=? ); ''' - self.c.execute(query, (inverter_serial,)) - first_data, last_data = self.c.fetchone() + self.cursor.execute(query, (inverter_serial,)) + first_data, last_data = self.cursor.fetchone() - if (first_data): data['hasPrevious'] = (first_data < day_start) - else: data['hasPrevious'] = False + if first_data: + data['hasPrevious'] = (first_data < day_start) + else: + data['hasPrevious'] = False - if (last_data): data['hasNext'] = (last_data > day_end) - else: data['hasNext'] = False + if last_data: + data['hasNext'] = (last_data > day_end) + else: + data['hasNext'] = False # print(json.dumps(data, indent=4)) return data @@ -217,11 +232,11 @@ def get_requested_month(self, date): ''' data['data'] = list() - for row in self.c.execute(query, (month_start, month_end)): + for row in self.cursor.execute(query, (month_start, month_end)): data['data'].append({'time': self.convert_local_ts_to_utc(row[0], self.local_timezone), 'power': row[1]}) month_total += row[1] - tot_start, tot_end = self.get_epoch_tot() + _, tot_end = self.get_epoch_tot() if date.split('-')[0] + '-' + date.split('-')[1] == datetime.utcfromtimestamp(tot_end).strftime('%Y-%m'): data['data'].append({'time': self.convert_local_ts_to_utc(tot_end, self.local_timezone), 'power': self.get_today()['dayTotal']}) month_total += self.get_today()['dayTotal'] @@ -233,13 +248,17 @@ def get_requested_month(self, date): FROM ( SELECT TimeStamp FROM MonthData GROUP BY TimeStamp ); ''' - self.c.execute(query) - first_data, last_data = self.c.fetchone() + self.cursor.execute(query) + first_data, last_data = self.cursor.fetchone() - if first_data: data['hasPrevious'] = (first_data < month_start) - else: data['hasPrevious'] = False - if last_data: data['hasNext'] = (last_data > month_end) - else: data['hasNext'] = False + if first_data: + data['hasPrevious'] = (first_data < month_start) + else: + data['hasPrevious'] = False + if last_data: + data['hasNext'] = (last_data > month_end) + else: + data['hasNext'] = False return data @@ -257,11 +276,11 @@ def get_requested_month_for_inverter(self, inverter_serial, date): ''' data['data'] = list() - for row in self.c.execute(query, (month_start, month_end, inverter_serial)): + for row in self.cursor.execute(query, (month_start, month_end, inverter_serial)): data['data'].append({'time': self.convert_local_ts_to_utc(row[0], self.local_timezone), 'power': row[1]}) month_total += row[1] - tot_start, tot_end = self.get_epoch_tot() + _, tot_end = self.get_epoch_tot() if date.split('-')[0] + '-' + date.split('-')[1] == datetime.utcfromtimestamp(tot_end).strftime('%Y-%m'): data['data'].append({'time': self.convert_local_ts_to_utc(tot_end, self.local_timezone), 'power': self.get_today()['inverters'][inverter_serial]['dayTotal']}) month_total += self.get_today()['inverters'][inverter_serial]['dayTotal'] @@ -274,13 +293,17 @@ def get_requested_month_for_inverter(self, inverter_serial, date): WHERE Serial=?; ''' - self.c.execute(query, (inverter_serial,)) - first_data, last_data = self.c.fetchone() + self.cursor.execute(query, (inverter_serial,)) + first_data, last_data = self.cursor.fetchone() - if first_data: data['hasPrevious'] = (first_data < month_start) - else: data['hasPrevious'] = False - if last_data: data['hasNext'] = (last_data > month_end) - else: data['hasNext'] = False + if first_data: + data['hasPrevious'] = (first_data < month_start) + else: + data['hasPrevious'] = False + if last_data: + data['hasNext'] = (last_data > month_end) + else: + data['hasNext'] = False return data @@ -307,7 +330,7 @@ def get_requested_year(self, date): current_month_start_local = self.convert_local_ts_to_utc(current_month_start, self.local_timezone) current_month_end_local = self.convert_local_ts_to_utc(current_month_end, self.local_timezone) - self.c.execute(query, (current_month_start_local, current_month_end_local)) + self.cursor.execute(query, (current_month_start_local, current_month_end_local)) gen_date = datetime(int(date.split('-')[0]), i, 1) month_total = self.get_requested_month(str(gen_date).split(' ')[0])['total'] @@ -329,13 +352,17 @@ def get_requested_year(self, date): FROM ( SELECT TimeStamp FROM MonthData GROUP BY TimeStamp ); ''' - self.c.execute(query) - first_data, last_data = self.c.fetchone() + self.cursor.execute(query) + first_data, last_data = self.cursor.fetchone() - if first_data: data['hasPrevious'] = (first_data < year_start) - else: data['hasPrevious'] = False - if last_data: data['hasNext'] = (last_data > year_end) - else: data['hasNext'] = False + if first_data: + data['hasPrevious'] = (first_data < year_start) + else: + data['hasPrevious'] = False + if last_data: + data['hasNext'] = (last_data > year_end) + else: + data['hasNext'] = False return data @@ -362,7 +389,7 @@ def get_requested_year_for_inverter(self, inverter_serial, date): current_month_start_local = self.convert_local_ts_to_utc(current_month_start, self.local_timezone) current_month_end_local = self.convert_local_ts_to_utc(current_month_end, self.local_timezone) - self.c.execute(query, (current_month_start_local, current_month_end_local, inverter_serial)) + self.cursor.execute(query, (current_month_start_local, current_month_end_local, inverter_serial)) gen_date = datetime(int(date.split('-')[0]), i, 1) month_total = self.get_requested_month_for_inverter(inverter_serial, str(gen_date).split(' ')[0])['total'] @@ -383,13 +410,17 @@ def get_requested_year_for_inverter(self, inverter_serial, date): FROM ( SELECT TimeStamp FROM MonthData WHERE Serial=? GROUP BY TimeStamp ); ''' - self.c.execute(query, (inverter_serial,)) - first_data, last_data = self.c.fetchone() + self.cursor.execute(query, (inverter_serial,)) + first_data, last_data = self.cursor.fetchone() - if first_data: data['hasPrevious'] = (first_data < year_start) - else: data['hasPrevious'] = False - if last_data: data['hasNext'] = (last_data > year_end) - else: data['hasNext'] = False + if first_data: + data['hasPrevious'] = (first_data < year_start) + else: + data['hasPrevious'] = False + if last_data: + data['hasNext'] = (last_data > year_end) + else: + data['hasNext'] = False return data @@ -406,9 +437,9 @@ def get_requested_tot(self): for i in range(int(string_start), int(string_end) + 1): gen_date = str(i) + "-01-01" - gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + gen_timestp = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) year = self.get_requested_year(gen_date) - data['data'].append({'time': gen_ts, 'power': year['total']}) + data['data'].append({'time': gen_timestp, 'power': year['total']}) return data @@ -425,9 +456,9 @@ def get_requested_tot_for_inverter(self, inverter_serial): for i in range(int(string_start), int(string_end) + 1): gen_date = str(i) + "-01-01" - gen_ts = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + gen_timestp = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) year = self.get_requested_year_for_inverter(inverter_serial, gen_date) - data['data'].append({'time': gen_ts, 'power': year['total']}) + data['data'].append({'time': gen_timestp, 'power': year['total']}) return data @@ -438,13 +469,13 @@ def get_inverters(self): ''' invs = [] renamings = self.config.get_renamings() - for row in self.c.execute(query): + for row in self.cursor.execute(query): serial = str(row[0]) name = row[1] if serial in renamings.keys(): name = renamings[serial] - invs.append( { + invs.append({ 'serial': serial, 'name': name, 'type': row[2], @@ -452,35 +483,35 @@ def get_inverters(self): 'etoday': row[4], 'etotal': row[5], 'status': row[6] - } ) + }) return invs def get_local_timezone(self): return datetime.now(tz=pytz.utc).astimezone().tzinfo - def convert_local_ts_to_utc(self, ts, local_timezone): - return int(datetime.utcfromtimestamp(ts).replace(tzinfo=local_timezone).timestamp()) + def convert_local_ts_to_utc(self, timestp, local_timezone): + return int(datetime.utcfromtimestamp(timestp).replace(tzinfo=local_timezone).timestamp()) def get_datetime(self, date): - s = date.split('-') - return datetime(int(s[0]), int(s[1]), int(s[2]), 00, 00, 00) + date_split = date.split('-') + return datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 00, 00, 00) def get_epoch_day(self, date): - s = date.split('-') - epoch_start = int(datetime(int(s[0]), int(s[1]), int(s[2]), 00, 00, 00, tzinfo=pytz.utc).timestamp()) - epoch_end = int(datetime(int(s[0]), int(s[1]), int(s[2]), 23, 59, 59, tzinfo=pytz.utc).timestamp()) + date_split = date.split('-') + epoch_start = int(datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 00, 00, 00, tzinfo=pytz.utc).timestamp()) + epoch_end = int(datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 23, 59, 59, tzinfo=pytz.utc).timestamp()) return epoch_start, epoch_end def get_epoch_month(self, date): - s = date.split('-') - epoch_start = int(datetime(int(s[0]), int(s[1]), 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) - epoch_end = int(datetime(int(s[0]), int(s[1]), self.get_last_day_of_month(date), 23, 59, 59, tzinfo=pytz.utc).timestamp()) + date_split = date.split('-') + epoch_start = int(datetime(int(date_split[0]), int(date_split[1]), 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + epoch_end = int(datetime(int(date_split[0]), int(date_split[1]), self.get_last_day_of_month(date), 23, 59, 59, tzinfo=pytz.utc).timestamp()) return epoch_start, epoch_end def get_epoch_year(self, date): - s = date.split('-') - epoch_start = int(datetime(int(s[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) - epoch_end = int(datetime(int(s[0]), 12, 31, 23, 59, 59, tzinfo=pytz.utc).timestamp()) + date_split = date.split('-') + epoch_start = int(datetime(int(date_split[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + epoch_end = int(datetime(int(date_split[0]), 12, 31, 23, 59, 59, tzinfo=pytz.utc).timestamp()) return epoch_start, epoch_end def get_epoch_tot(self): @@ -489,28 +520,28 @@ def get_epoch_tot(self): FROM ( SELECT TimeStamp FROM MonthData GROUP BY TimeStamp ); ''' - self.c.execute(query) - epoch_start, epoch_end = self.c.fetchone() + self.cursor.execute(query) + epoch_start, epoch_end = self.cursor.fetchone() query = ''' SELECT MIN(TimeStamp) as Min, MAX(TimeStamp) as Max FROM ( SELECT TimeStamp FROM DayData GROUP BY TimeStamp ); ''' - self.c.execute(query) - day_start, day_end = self.c.fetchone() + self.cursor.execute(query) + _, day_end = self.cursor.fetchone() if day_end > epoch_end: epoch_end = day_end return epoch_start, epoch_end def get_epoch_month_ends_for_year(self, date): - s = date.split('-') + date_split = date.split('-') month_ends = [] - for i in range (1,13): - last_day_for_month = self.get_last_day_of_month(s[0] + "-" + "{:02d}".format(i) + "-" + "01") - ts = int(datetime(int(s[0]), i, last_day_for_month, 23, 59, 59, tzinfo=pytz.utc).timestamp()) - month_ends.append(ts) + for i in range(1, 13): + last_day_for_month = self.get_last_day_of_month(date_split[0] + "-" + "{:02d}".format(i) + "-" + "01") + timestp = int(datetime(int(date_split[0]), i, last_day_for_month, 23, 59, 59, tzinfo=pytz.utc).timestamp()) + month_ends.append(timestp) return month_ends def get_last_day_of_month(self, date): @@ -519,7 +550,7 @@ def get_last_day_of_month(self, date): return (next_month - timedelta(days=next_month.day)).day def close(self): - self.db.close() + self.database.close() if __name__ == '__main__': From a255366e26af243f1fe6251482edad1f476281cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Fri, 16 Aug 2019 01:37:12 +0200 Subject: [PATCH 10/11] Extract time functions from database.px, minimize pylint hints --- static/js/main.js | 4 +- util/database.py | 206 +++++++++++++++++++++++----------------------- util/time.py | 88 ++++++++++++++++++++ 3 files changed, 192 insertions(+), 106 deletions(-) create mode 100644 util/time.py diff --git a/static/js/main.js b/static/js/main.js index 22e611f..6c65deb 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -571,8 +571,8 @@ function loadData(load_mode = 1) { $("#chart-tot-col .inverter-yield").text( addPrefix(response.today.total) + "Wh"); // update scale - all.tot.interval.from = moment.unix(all.tot.interval.from).subtract('years', 1); - all.tot.interval.to = moment.unix(all.tot.interval.to).add('years', 1); + all.tot.interval.from = moment.unix(all.tot.interval.from).subtract(1, 'years'); + all.tot.interval.to = moment.unix(all.tot.interval.to).add(1, 'years'); totChart.data.labels = [ all.tot.interval.from, all.tot.interval.to diff --git a/util/database.py b/util/database.py index bfc1d70..f97dd6f 100644 --- a/util/database.py +++ b/util/database.py @@ -1,12 +1,11 @@ #!/usr/bin/python3 -""" -""" +"""Process SBFspot database to json format""" import sqlite3 -from datetime import datetime, timedelta -import pytz +import util.time as time class Database(): + """The class""" def __init__(self, config): self.config = config @@ -14,22 +13,24 @@ def __init__(self, config): self.database = sqlite3.connect(self.config.get_database_path(), check_same_thread=False) self.cursor = self.database.cursor() - self.local_timezone = self.get_local_timezone() + self.local_timezone = time.get_local_timezone() def get(self, date, requested_data): + """Function called from outside""" tot_start, tot_end = self.get_epoch_tot() date_split = date.split('-') - int_date = int(datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 00, 00, 00, tzinfo=pytz.utc).timestamp()) + int_date = time.get_start_of_day(date_split[0], date_split[1], date_split[2]) if int_date < tot_start: - date = datetime.utcfromtimestamp(tot_start).strftime('%Y-%m-%d') + date = time.get_datestring(tot_start, '%Y-%m-%d') if int_date > tot_end: - date = datetime.utcfromtimestamp(tot_end).strftime('%Y-%m-%d') + date = time.get_datestring(tot_end, '%Y-%m-%d') data = dict() data['today'] = self.get_today() data['requested'] = self.get_requested(date, requested_data) return data def get_today(self): + """Process data stored in today's section""" data = dict() total_day = 0 @@ -67,6 +68,7 @@ def get_today(self): return data def get_requested(self, date, requested_data): + """Process requested data""" data = dict() data['date'] = date @@ -87,22 +89,29 @@ def get_requested(self, date, requested_data): data['inverters'][inv['serial']] = {'day': [], 'month': [], 'year': [], 'tot': []} if requested_data['day'] is True: - data['inverters'][inv['serial']]['day'] = self.get_requested_day_for_inverter(inv['serial'], date) + day_data = self.get_requested_day_for_inverter(inv['serial'], date) + data['inverters'][inv['serial']]['day'] = day_data if requested_data['month'] is True: - data['inverters'][inv['serial']]['month'] = self.get_requested_month_for_inverter(inv['serial'], date) + month_data = self.get_requested_month_for_inverter(inv['serial'], date) + data['inverters'][inv['serial']]['month'] = month_data if requested_data['year'] is True: - data['inverters'][inv['serial']]['year'] = self.get_requested_year_for_inverter(inv['serial'], date) + year_data = self.get_requested_year_for_inverter(inv['serial'], date) + data['inverters'][inv['serial']]['year'] = year_data if requested_data['tot'] is True: - data['inverters'][inv['serial']]['tot'] = self.get_requested_tot_for_inverter(inv['serial']) + tot_data = self.get_requested_tot_for_inverter(inv['serial']) + data['inverters'][inv['serial']]['tot'] = tot_data return data def get_requested_day(self, date): + """Process overall day data""" data = dict() - day_start, day_end = self.get_epoch_day(date) - data['interval'] = {'from': self.convert_local_ts_to_utc(day_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(day_end, self.local_timezone)} + day_start, day_end = time.get_epoch_day(date) + from_ts = time.convert_local_ts_to_utc(day_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(day_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} query = ''' SELECT TimeStamp, SUM(Power) AS Power @@ -116,7 +125,7 @@ def get_requested_day(self, date): data['data'].append({'time': row[0], 'power': row[1]}) - if self.get_datetime(date).date() == datetime.today().date(): + if time.get_is_today(date): query = ''' SELECT SUM(EToday) as EToday FROM Inverters; @@ -160,10 +169,13 @@ def get_requested_day(self, date): return data def get_requested_day_for_inverter(self, inverter_serial, date): + """Process day data per inverter""" data = dict() - day_start, day_end = self.get_epoch_day(date) - data['interval'] = {'from': self.convert_local_ts_to_utc(day_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(day_end, self.local_timezone)} + day_start, day_end = time.get_epoch_day(date) + from_ts = time.convert_local_ts_to_utc(day_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(day_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} query = ''' SELECT TimeStamp, Power @@ -175,7 +187,7 @@ def get_requested_day_for_inverter(self, inverter_serial, date): for row in self.cursor.execute(query, (day_start, day_end, inverter_serial)): data['data'].append({'time': row[0], 'power': row[1]}) - if self.get_datetime(date).date() == datetime.today().date(): + if time.get_is_today(date): query = ''' SELECT EToday FROM Inverters @@ -218,10 +230,13 @@ def get_requested_day_for_inverter(self, inverter_serial, date): return data def get_requested_month(self, date): + """Process overall month data""" data = dict() - month_start, month_end = self.get_epoch_month(date) - data['interval'] = {'from': self.convert_local_ts_to_utc(month_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(month_end, self.local_timezone)} + month_start, month_end = time.get_epoch_month(date) + from_ts = time.convert_local_ts_to_utc(month_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(month_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} month_total = 0 query = ''' @@ -233,12 +248,15 @@ def get_requested_month(self, date): data['data'] = list() for row in self.cursor.execute(query, (month_start, month_end)): - data['data'].append({'time': self.convert_local_ts_to_utc(row[0], self.local_timezone), 'power': row[1]}) + time_ts = time.convert_local_ts_to_utc(row[0], self.local_timezone) + data['data'].append({'time': time_ts, 'power': row[1]}) month_total += row[1] _, tot_end = self.get_epoch_tot() - if date.split('-')[0] + '-' + date.split('-')[1] == datetime.utcfromtimestamp(tot_end).strftime('%Y-%m'): - data['data'].append({'time': self.convert_local_ts_to_utc(tot_end, self.local_timezone), 'power': self.get_today()['dayTotal']}) + if date.split('-')[0] + '-' + date.split('-')[1] == time.get_datestring(tot_end, '%Y-%m'): + time_ts = time.convert_local_ts_to_utc(tot_end, self.local_timezone) + power_over = self.get_today()['dayTotal'] + data['data'].append({'time': time_ts, 'power': power_over}) month_total += self.get_today()['dayTotal'] data['total'] = month_total @@ -263,10 +281,13 @@ def get_requested_month(self, date): return data def get_requested_month_for_inverter(self, inverter_serial, date): + """Process month data per inverter""" data = dict() - month_start, month_end = self.get_epoch_month(date) - data['interval'] = {'from': self.convert_local_ts_to_utc(month_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(month_end, self.local_timezone)} + month_start, month_end = time.get_epoch_month(date) + from_ts = time.convert_local_ts_to_utc(month_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(month_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} month_total = 0 query = ''' @@ -277,12 +298,15 @@ def get_requested_month_for_inverter(self, inverter_serial, date): data['data'] = list() for row in self.cursor.execute(query, (month_start, month_end, inverter_serial)): - data['data'].append({'time': self.convert_local_ts_to_utc(row[0], self.local_timezone), 'power': row[1]}) + time_ts = time.convert_local_ts_to_utc(row[0], self.local_timezone) + data['data'].append({'time': time_ts, 'power': row[1]}) month_total += row[1] _, tot_end = self.get_epoch_tot() - if date.split('-')[0] + '-' + date.split('-')[1] == datetime.utcfromtimestamp(tot_end).strftime('%Y-%m'): - data['data'].append({'time': self.convert_local_ts_to_utc(tot_end, self.local_timezone), 'power': self.get_today()['inverters'][inverter_serial]['dayTotal']}) + if date.split('-')[0] + '-' + date.split('-')[1] == time.get_datestring(tot_end, '%Y-%m'): + time_ts = time.convert_local_ts_to_utc(tot_end, self.local_timezone) + power_inv = self.get_today()['inverters'][inverter_serial]['dayTotal'] + data['data'].append({'time': time_ts, 'power': power_inv}) month_total += self.get_today()['inverters'][inverter_serial]['dayTotal'] data['total'] = month_total @@ -308,13 +332,16 @@ def get_requested_month_for_inverter(self, inverter_serial, date): return data def get_requested_year(self, date): + """Process overall year data""" data = dict() - year_start, year_end = self.get_epoch_year(date) - data['interval'] = {'from': self.convert_local_ts_to_utc(year_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(year_end, self.local_timezone)} + year_start, year_end = time.get_epoch_year(date) + from_ts = time.convert_local_ts_to_utc(year_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(year_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} data['data'] = list() - month_ends = self.get_epoch_month_ends_for_year(date) + month_ends = time.get_epoch_month_ends_for_year(date) query = ''' SELECT SUM(DayYield) AS Power @@ -322,28 +349,28 @@ def get_requested_year(self, date): WHERE TimeStamp BETWEEN ? AND ?; ''' - current_month_start = year_start - current_month_end = month_ends[0] + month_start = year_start + month_end = month_ends[0] year_total = 0 for i in range(1, 13): - current_month_start_local = self.convert_local_ts_to_utc(current_month_start, self.local_timezone) - current_month_end_local = self.convert_local_ts_to_utc(current_month_end, self.local_timezone) + month_start_local = time.convert_local_ts_to_utc(month_start, self.local_timezone) + month_end_local = time.convert_local_ts_to_utc(month_end, self.local_timezone) - self.cursor.execute(query, (current_month_start_local, current_month_end_local)) + self.cursor.execute(query, (month_start_local, month_end_local)) - gen_date = datetime(int(date.split('-')[0]), i, 1) - month_total = self.get_requested_month(str(gen_date).split(' ')[0])['total'] + gen_date = str(time.convert_ts_to_date(date.split('-')[0], i, 1)) + month_total = self.get_requested_month(gen_date.split(' ')[0])['total'] if month_total is None: month_total = 0 year_total += month_total - data['data'].append({'time': current_month_start, 'power': month_total}) + data['data'].append({'time': month_start, 'power': month_total}) if i < 12: - current_month_start = current_month_end+1 - current_month_end = month_ends[i] + month_start = month_end+1 + month_end = month_ends[i] data['total'] = year_total @@ -367,13 +394,16 @@ def get_requested_year(self, date): return data def get_requested_year_for_inverter(self, inverter_serial, date): + """Process year data per inverter""" data = dict() - year_start, year_end = self.get_epoch_year(date) - data['interval'] = {'from': self.convert_local_ts_to_utc(year_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(year_end, self.local_timezone)} + year_start, year_end = time.get_epoch_year(date) + from_ts = time.convert_local_ts_to_utc(year_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(year_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} data['data'] = list() - month_ends = self.get_epoch_month_ends_for_year(date) + month_ends = time.get_epoch_month_ends_for_year(date) query = ''' SELECT SUM(DayYield) AS Power @@ -381,27 +411,28 @@ def get_requested_year_for_inverter(self, inverter_serial, date): WHERE TimeStamp BETWEEN ? AND ? AND Serial=?; ''' - current_month_start = year_start - current_month_end = month_ends[0] + month_start = year_start + month_end = month_ends[0] year_total = 0 for i in range(1, 13): - current_month_start_local = self.convert_local_ts_to_utc(current_month_start, self.local_timezone) - current_month_end_local = self.convert_local_ts_to_utc(current_month_end, self.local_timezone) + month_start_local = time.convert_local_ts_to_utc(month_start, self.local_timezone) + month_end_local = time.convert_local_ts_to_utc(month_end, self.local_timezone) - self.cursor.execute(query, (current_month_start_local, current_month_end_local, inverter_serial)) + self.cursor.execute(query, (month_start_local, month_end_local, inverter_serial)) - gen_date = datetime(int(date.split('-')[0]), i, 1) - month_total = self.get_requested_month_for_inverter(inverter_serial, str(gen_date).split(' ')[0])['total'] + gen_date = str(time.convert_ts_to_date(date.split('-')[0], i, 1)) + out = self.get_requested_month_for_inverter(inverter_serial, gen_date.split(' ')[0]) + month_total = out['total'] if month_total is None: month_total = 0 year_total += month_total - data['data'].append({'time': current_month_start, 'power': month_total}) + data['data'].append({'time': month_start, 'power': month_total}) if i < 12: - current_month_start = current_month_end+1 - current_month_end = month_ends[i] + month_start = month_end+1 + month_end = month_ends[i] data['total'] = year_total @@ -425,44 +456,51 @@ def get_requested_year_for_inverter(self, inverter_serial, date): return data def get_requested_tot(self): + """Process overall total data""" data = dict() tot_start, tot_end = self.get_epoch_tot() - data['interval'] = {'from': self.convert_local_ts_to_utc(tot_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(tot_end, self.local_timezone)} + from_ts = time.convert_local_ts_to_utc(tot_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(tot_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} - string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y') - string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y') + string_start = time.get_datestring(tot_start, '%Y') + string_end = time.get_datestring(tot_end, '%Y') data['data'] = list() for i in range(int(string_start), int(string_end) + 1): gen_date = str(i) + "-01-01" - gen_timestp = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + gen_timestp = time.get_start_of_day(gen_date.split('-')[0], 1, 1) year = self.get_requested_year(gen_date) data['data'].append({'time': gen_timestp, 'power': year['total']}) return data def get_requested_tot_for_inverter(self, inverter_serial): + """Process total data per inverter""" data = dict() tot_start, tot_end = self.get_epoch_tot() - data['interval'] = {'from': self.convert_local_ts_to_utc(tot_start, self.local_timezone), 'to': self.convert_local_ts_to_utc(tot_end, self.local_timezone)} + from_ts = time.convert_local_ts_to_utc(tot_start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(tot_end, self.local_timezone) + data['interval'] = {'from': from_ts, 'to': to_ts} - string_start = datetime.utcfromtimestamp(tot_start).strftime('%Y') - string_end = datetime.utcfromtimestamp(tot_end).strftime('%Y') + string_start = time.get_datestring(tot_start, '%Y') + string_end = time.get_datestring(tot_end, '%Y') data['data'] = list() for i in range(int(string_start), int(string_end) + 1): gen_date = str(i) + "-01-01" - gen_timestp = int(datetime(int(gen_date.split('-')[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) + gen_timestp = time.get_start_of_day(gen_date.split('-')[0], 1, 1) year = self.get_requested_year_for_inverter(inverter_serial, gen_date) data['data'].append({'time': gen_timestp, 'power': year['total']}) return data def get_inverters(self): + """Detect inverters from database""" query = ''' SELECT Serial, Name, Type, TimeStamp, EToday, ETotal, Status, OperatingTime FROM Inverters; @@ -486,35 +524,8 @@ def get_inverters(self): }) return invs - def get_local_timezone(self): - return datetime.now(tz=pytz.utc).astimezone().tzinfo - - def convert_local_ts_to_utc(self, timestp, local_timezone): - return int(datetime.utcfromtimestamp(timestp).replace(tzinfo=local_timezone).timestamp()) - - def get_datetime(self, date): - date_split = date.split('-') - return datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 00, 00, 00) - - def get_epoch_day(self, date): - date_split = date.split('-') - epoch_start = int(datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 00, 00, 00, tzinfo=pytz.utc).timestamp()) - epoch_end = int(datetime(int(date_split[0]), int(date_split[1]), int(date_split[2]), 23, 59, 59, tzinfo=pytz.utc).timestamp()) - return epoch_start, epoch_end - - def get_epoch_month(self, date): - date_split = date.split('-') - epoch_start = int(datetime(int(date_split[0]), int(date_split[1]), 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) - epoch_end = int(datetime(int(date_split[0]), int(date_split[1]), self.get_last_day_of_month(date), 23, 59, 59, tzinfo=pytz.utc).timestamp()) - return epoch_start, epoch_end - - def get_epoch_year(self, date): - date_split = date.split('-') - epoch_start = int(datetime(int(date_split[0]), 1, 1, 00, 00, 00, tzinfo=pytz.utc).timestamp()) - epoch_end = int(datetime(int(date_split[0]), 12, 31, 23, 59, 59, tzinfo=pytz.utc).timestamp()) - return epoch_start, epoch_end - def get_epoch_tot(self): + """Detect first and last timestamp from database""" query = ''' SELECT MIN(TimeStamp) as Min, MAX(TimeStamp) as Max FROM ( SELECT TimeStamp FROM MonthData GROUP BY TimeStamp ); @@ -535,21 +546,8 @@ def get_epoch_tot(self): return epoch_start, epoch_end - def get_epoch_month_ends_for_year(self, date): - date_split = date.split('-') - month_ends = [] - for i in range(1, 13): - last_day_for_month = self.get_last_day_of_month(date_split[0] + "-" + "{:02d}".format(i) + "-" + "01") - timestp = int(datetime(int(date_split[0]), i, last_day_for_month, 23, 59, 59, tzinfo=pytz.utc).timestamp()) - month_ends.append(timestp) - return month_ends - - def get_last_day_of_month(self, date): - day = datetime.strptime(date, "%Y-%m-%d") - next_month = day.replace(day=28) + timedelta(days=4) # this will never fail - return (next_month - timedelta(days=next_month.day)).day - def close(self): + """Close database connection""" self.database.close() if __name__ == '__main__': diff --git a/util/time.py b/util/time.py new file mode 100644 index 0000000..e23fca6 --- /dev/null +++ b/util/time.py @@ -0,0 +1,88 @@ +#!/usr/bin/python3 +"""Time calculation functions""" +from datetime import datetime, timedelta +import pytz + + +def get_local_timezone(): + """Get local timezone""" + return datetime.now(tz=pytz.utc).astimezone().tzinfo + +def convert_local_ts_to_utc(timestp, local_timezone): + """Convert local timestamp to utc timestamp""" + return int(datetime.utcfromtimestamp(timestp).replace(tzinfo=local_timezone).timestamp()) + +def convert_ts_to_date(year, month, day): + """Convert a complete timestamp to a date timestamp""" + return datetime(int(year), int(month), int(day)) + +def get_datetime(date): + """Convert a date to timestamp""" + date_split = date.split('-') + return get_start_of_day(date_split[0], date_split[1], date_split[2]) + +def get_epoch_day(date): + """Output a day's time range""" + date_split = date.split('-') + epoch_start = get_start_of_day(date_split[0], date_split[1], date_split[2]) + epoch_end = get_end_of_day(date_split[0], date_split[1], date_split[2]) + return epoch_start, epoch_end + +def get_epoch_month(date): + """Output a month's time range""" + date_split = date.split('-') + epoch_start = get_start_of_day(date_split[0], date_split[1], 1) + day = get_last_day_of_month(date) + epoch_end = get_end_of_day(date_split[0], date_split[1], day) + return epoch_start, epoch_end + +def get_epoch_year(date): + """Output a year's time range""" + date_split = date.split('-') + epoch_start = get_start_of_day(date_split[0], 1, 1) + epoch_end = get_end_of_day(date_split[0], 12, 31) + return epoch_start, epoch_end + +def get_epoch_month_ends_for_year(date): + """Calculate all end timestamps of each months of a year""" + date_split = date.split('-') + month_ends = [] + for i in range(1, 13): + day = date_split[0] + "-" + "{:02d}".format(i) + "-" + "01" + last_day_for_month = get_last_day_of_month(day) + timestp = get_end_of_day(int(date_split[0]), i, last_day_for_month) + month_ends.append(timestp) + return month_ends + +def get_last_day_of_month(date): + """Get the last day of a month""" + day = datetime.strptime(date, "%Y-%m-%d") + next_month = day.replace(day=28) + timedelta(days=4) # this will never fail + return (next_month - timedelta(days=next_month.day)).day + +def get_start_of_day(year, month, day): + """Get the day's begin timestamp""" + datetime_start = datetime(int(year), int(month), int(day), 00, 00, 00, tzinfo=pytz.utc) + day_start = int(datetime_start.timestamp()) + return day_start + +def get_end_of_day(year, month, day): + """Get the day's end timestamp""" + datetime_end = datetime(int(year), int(month), int(day), 23, 59, 59, tzinfo=pytz.utc) + day_end = int(datetime_end.timestamp()) + return day_end + +def get_datestring(timestp, strformat): + """Convert a timestamp to a date string""" + date = datetime.utcfromtimestamp(timestp).strftime(strformat) + return date + +def get_is_today(date): + """Check if parsed date is today's date""" + today_date = datetime.today().date() + return date == today_date + + +if __name__ == '__main__': + + print("nothing to do here") From fd28cb97057c70ccb8c52c2cd3ea06e74c8f707e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjarne=20Ro=C3=9F?= Date: Fri, 16 Aug 2019 13:14:56 +0200 Subject: [PATCH 11/11] Further PEP8ing --- sunportal.py | 34 ++++++++++++-------- util/config.py | 40 ++++++++++++++--------- util/database.py | 40 +++++++++-------------- util/mail.py | 84 ++++++++++++++++++++++++++++++------------------ 4 files changed, 112 insertions(+), 86 deletions(-) diff --git a/sunportal.py b/sunportal.py index e2e1e4b..09e282e 100755 --- a/sunportal.py +++ b/sunportal.py @@ -14,12 +14,12 @@ from util.database import Database from util.mail import Mail -config = Config() -db = Database(config=config) -mail = Mail(config, db) -app = Flask(__name__) +CONFIG = Config() +DB = Database(config=CONFIG) +MAIL = Mail(CONFIG, DB) +APP = Flask(__name__) -schema = { +SCHEMA = { 'type': 'object', 'properties': { 'date': { @@ -40,23 +40,29 @@ } -@app.route('/') +@APP.route('/') def home(): + """Set html file""" return render_template('index.html') -@app.route('/update', methods=['POST']) -@expects_json(schema) +@APP.route('/update', methods=['POST']) +@expects_json(SCHEMA) def update(): + """Do the request""" + val = jsonify() if request.headers['Content-Type'] == 'application/json': content = request.json - if 'date' not in content: flask.abort(400) - if 'requested_data' not in content: flask.abort(400) + if 'date' not in content: + flask.abort(400) + if 'requested_data' not in content: + flask.abort(400) date = content['date'] requested_data = content['requested_data'] - return jsonify(db.get(date, requested_data)) - flask.abort(400) + val = jsonify(DB.get(date, requested_data)) + return val if __name__ == '__main__': - if mail.is_enabled: mail.start() - app.run(host='0.0.0.0', port=80) + if MAIL.is_enabled: + MAIL.start() + APP.run(host='0.0.0.0', port=80) diff --git a/util/config.py b/util/config.py index c8768c5..cab90ad 100644 --- a/util/config.py +++ b/util/config.py @@ -3,8 +3,8 @@ Config module Reads configuration file """ -import yaml import os.path +import yaml class Config(): """The Config class loads the config YAML file""" @@ -13,43 +13,51 @@ def __init__(self, config_path=None): self.config = dict() if config_path: - with open(config_path) as f: - self.config = yaml.load(f) + with open(config_path) as file: + self.config = yaml.load(file) else: try: - with open('config.yml') as f: - self.config = yaml.load(f) + with open('config.yml') as file: + self.config = yaml.load(file) except OSError: - with open('config.default.yml') as f: - self.config = yaml.load(f) + with open('config.default.yml') as file: + self.config = yaml.load(file) def get_config(self): + """Fetch basic configs""" return self.config def get_mail_config(self): + """Fetch mail configs""" return self.config["mail"] def get_database_path(self): + """Fetch database path from config""" path = self.config["database"]["path"] - if os.path.isfile(path): - return self.config["database"]["path"] - else: + if not os.path.isfile(path): msg = "sqlite database %s does not exist, check the config(.default).json!" % path raise Exception(msg) + return path def get_co2_avoidance_factor(self): + """Fetch CO2 avoidance factor from config""" return self.config["co2_avoidance_factor"] def get_renamings(self): + """Fetch inverter's name from config""" return self.config["renaming"] - def log(self, msg, error=''): - if error: print(' *', msg, '['+str(error)+']') - else: print(' *', msg) + @staticmethod + def log(msg, error=''): + """Define logging""" + if error: + print(' *', msg, '['+str(error)+']') + else: + print(' *', msg) if __name__ == '__main__': - cfg = Config(config_path='../config.yml') - print(cfg.get_config()) - print(cfg.get_co2_avoidance_factor()) + CFG = Config(config_path='../config.yml') + print(CFG.get_config()) + print(CFG.get_co2_avoidance_factor()) diff --git a/util/database.py b/util/database.py index f97dd6f..7898f47 100644 --- a/util/database.py +++ b/util/database.py @@ -5,7 +5,7 @@ class Database(): - """The class""" + """Database class""" def __init__(self, config): self.config = config @@ -109,9 +109,7 @@ def get_requested_day(self, date): data = dict() day_start, day_end = time.get_epoch_day(date) - from_ts = time.convert_local_ts_to_utc(day_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(day_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(day_start, day_end) query = ''' SELECT TimeStamp, SUM(Power) AS Power @@ -173,9 +171,7 @@ def get_requested_day_for_inverter(self, inverter_serial, date): data = dict() day_start, day_end = time.get_epoch_day(date) - from_ts = time.convert_local_ts_to_utc(day_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(day_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(day_start, day_end) query = ''' SELECT TimeStamp, Power @@ -234,9 +230,7 @@ def get_requested_month(self, date): data = dict() month_start, month_end = time.get_epoch_month(date) - from_ts = time.convert_local_ts_to_utc(month_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(month_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(month_start, month_end) month_total = 0 query = ''' @@ -285,9 +279,7 @@ def get_requested_month_for_inverter(self, inverter_serial, date): data = dict() month_start, month_end = time.get_epoch_month(date) - from_ts = time.convert_local_ts_to_utc(month_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(month_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(month_start, month_end) month_total = 0 query = ''' @@ -336,9 +328,7 @@ def get_requested_year(self, date): data = dict() year_start, year_end = time.get_epoch_year(date) - from_ts = time.convert_local_ts_to_utc(year_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(year_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(year_start, year_end) data['data'] = list() month_ends = time.get_epoch_month_ends_for_year(date) @@ -398,9 +388,7 @@ def get_requested_year_for_inverter(self, inverter_serial, date): data = dict() year_start, year_end = time.get_epoch_year(date) - from_ts = time.convert_local_ts_to_utc(year_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(year_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(year_start, year_end) data['data'] = list() month_ends = time.get_epoch_month_ends_for_year(date) @@ -460,9 +448,7 @@ def get_requested_tot(self): data = dict() tot_start, tot_end = self.get_epoch_tot() - from_ts = time.convert_local_ts_to_utc(tot_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(tot_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(tot_start, tot_end) string_start = time.get_datestring(tot_start, '%Y') string_end = time.get_datestring(tot_end, '%Y') @@ -482,9 +468,7 @@ def get_requested_tot_for_inverter(self, inverter_serial): data = dict() tot_start, tot_end = self.get_epoch_tot() - from_ts = time.convert_local_ts_to_utc(tot_start, self.local_timezone) - to_ts = time.convert_local_ts_to_utc(tot_end, self.local_timezone) - data['interval'] = {'from': from_ts, 'to': to_ts} + data['interval'] = self.get_interval(tot_start, tot_end) string_start = time.get_datestring(tot_start, '%Y') string_end = time.get_datestring(tot_end, '%Y') @@ -546,6 +530,12 @@ def get_epoch_tot(self): return epoch_start, epoch_end + def get_interval(self, start, end): + """Get specific chart interval converting timestamp to utc""" + from_ts = time.convert_local_ts_to_utc(start, self.local_timezone) + to_ts = time.convert_local_ts_to_utc(end, self.local_timezone) + return {'from': from_ts, 'to': to_ts} + def close(self): """Close database connection""" self.database.close() diff --git a/util/mail.py b/util/mail.py index 42a8a8d..434f9a3 100644 --- a/util/mail.py +++ b/util/mail.py @@ -1,17 +1,22 @@ #!/usr/bin/python3 """ +Mail module +Manages mailing """ -import smtplib, threading, time +import smtplib +import threading +import time from datetime import datetime, timedelta from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart class Mail: + """Mail class""" def __init__(self, config, database): self.config = config - self.db = database + self.database = database self.mail_config = self.config.get_mail_config() self.is_enabled = self.mail_config["enabled"] == 'true' self.timer_thread = None @@ -19,19 +24,22 @@ def __init__(self, config, database): self.sent_messages = dict() def start(self): - self.timer_thread = threading.Thread(target=self.run) # run self.run every [interval] seconds + """Start the mailing service""" + # run self.run every [interval] seconds + self.timer_thread = threading.Thread(target=self.run) self.timer_thread.start() self.config.log('mail: service started') def run(self): - interval = self.mail_config["check_interval"] # in seconds + """Do the mailing procedure""" + interval = self.mail_config["check_interval"] # in seconds while not self.stop_threads: - inverters = self.db.get_inverters() + inverters = self.database.get_inverters() for inv in inverters: # if status is not okay if inv['status'] != 'OK' and self.do_send_mail('status', inv['serial']): - msg = 'Inverter \'%s\' [Serial: %s] has abnormal status: \'%s\'' % ( inv['name'], inv['serial'], inv['status']) + msg = 'Inverter \'%s\' [Serial: %s] has abnormal status: \'%s\'' % (inv['name'], inv['serial'], inv['status']) self.config.log('mail: sending error mail', msg) message_id = '%s-%s-%s' % (datetime.now().timestamp(), inv['serial'], 'status') self.send_mail('Abnormal inverter status', msg, message_id=message_id) @@ -46,24 +54,24 @@ def run(self): # if last timestamp is older than 24h older_than_24h = datetime.fromtimestamp(inv['ts']) < (datetime.now() - timedelta(days=1)) if older_than_24h and self.do_send_mail('timeout', inv['serial']): - ts = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.fromtimestamp(inv['ts'])) - msg = 'Last update from inverter \'%s\' [Serial: %s] is more than 24 hours ago, last timestamp: %s' % ( - inv['name'], inv['serial'], ts) + timestp = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.fromtimestamp(inv['ts'])) + msg = 'Last update from inverter \'%s\' [Serial: %s] is more than 24 hours ago, last timestamp: %s' % (inv['name'], inv['serial'], timestp) self.config.log('mail: sending error mail', msg) message_id = '%s-%s-%s' % (datetime.now().timestamp(), inv['serial'], 'timeout') self.send_mail('Last inverter data older than 24h', msg, message_id=message_id) - self.set_sent_mail('timeout', inv['serial'], message_id) + self.set_sent_mail('timeout', inv['serial'], message_id) elif not older_than_24h and self.do_send_clear_mail('timeout', inv['serial']): - ts = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.fromtimestamp(inv['ts'])) - msg = 'Inverter \'%s\' [Serial: %s] now delivers values again, last timestamp: %s' % (inv['name'], inv['serial'], ts) + timestp = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.fromtimestamp(inv['ts'])) + msg = 'Inverter \'%s\' [Serial: %s] now delivers values again, last timestamp: %s' % (inv['name'], inv['serial'], timestp) self.config.log('mail: sending back to normal mail', msg) message_id = self.get_message_id('timeout', inv['serial']) self.send_mail('Last inverter data older than 24h', msg, message_id=message_id, in_reply_to=True) self.set_sent_mail('timeout', inv['serial'], message_id, sent=False) time.sleep(interval) - def send_mail(self, subject, message, message_id=None, in_reply_to=False, debug=False ): - if len(self.mail_config["recipients"]) > 0: + def send_mail(self, subject, message, message_id=None, in_reply_to=False, debug=False): + """Send a mail""" + if self.mail_config["recipients"]: smtp_server = self.mail_config["smtp_server"] sender = self.mail_config["sender"] recipients = self.mail_config["recipients"] @@ -89,15 +97,18 @@ def send_mail(self, subject, message, message_id=None, in_reply_to=False, debug= # image = add_header('Content-ID', '') # msg.attach(image) - with smtplib.SMTP(smtp_server["url"], port=smtp_server["port"]) as s: - if debug: s.set_debuglevel(1) + with smtplib.SMTP(smtp_server["url"], port=smtp_server["port"]) as server: + if debug: + server.set_debuglevel(1) if starttls and starttls["enabled"]: - s.starttls() - s.login(starttls["user"], starttls["password"]) - s.sendmail(sender, recipients, msg.as_string()) + server.starttls() + server.login(starttls["user"], starttls["password"]) + server.sendmail(sender, recipients, msg.as_string()) def set_sent_mail(self, msg_type, identifier, message_id, sent=True): - if msg_type not in self.sent_messages: self.sent_messages[msg_type] = {identifier: dict()} + """Actions when mail is sent""" + if msg_type not in self.sent_messages: + self.sent_messages[msg_type] = {identifier: dict()} if sent: self.sent_messages[msg_type][identifier] = { 'action': 'sent', @@ -112,8 +123,11 @@ def set_sent_mail(self, msg_type, identifier, message_id, sent=True): } def do_send_mail(self, msg_type, identifier): - if msg_type not in self.sent_messages: return True - if identifier not in self.sent_messages[msg_type]: return True + """Return mail send status""" + if msg_type not in self.sent_messages: + return True + if identifier not in self.sent_messages[msg_type]: + return True i = self.sent_messages[msg_type][identifier] if i['action'] == 'sent' and i['time'] < (datetime.now() - timedelta(hours=24)): return True @@ -122,18 +136,26 @@ def do_send_mail(self, msg_type, identifier): return False def do_send_clear_mail(self, msg_type, identifier): - if msg_type not in self.sent_messages: return False - if identifier not in self.sent_messages[msg_type]: return False + """Return clear mail send status""" + if msg_type not in self.sent_messages: + return False + if identifier not in self.sent_messages[msg_type]: + return False i = self.sent_messages[msg_type][identifier] - if i['action'] == 'sent': return True + if i['action'] == 'sent': + return True return False def get_message_id(self, msg_type, identifier): - if msg_type not in self.sent_messages: return None - if identifier not in self.sent_messages[msg_type]: return None + """Get message id""" + if msg_type not in self.sent_messages: + return None + if identifier not in self.sent_messages[msg_type]: + return None return self.sent_messages[msg_type][identifier]['message_id'] def join(self): + """Join function""" self.stop_threads = True if __name__ == '__main__': @@ -141,8 +163,8 @@ def join(self): from util.database import Database from util.config import Config - cfg = Config(config_path='../config.yml') - db = Database(cfg) + CFG = Config(config_path='../config.yml') + DB = Database(CFG) - mail = Mail(cfg, db) - mail.send_mail('test mail', 'this is your test mail', debug=True) + MAIL = Mail(CFG, DB) + MAIL.send_mail('test mail', 'this is your test mail', debug=True)