From 54548d896d874ca8731ad6769157ce6691507701 Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Sun, 1 Mar 2020 12:34:25 -0800 Subject: [PATCH] add a script that generates fake data.. definitely needs work... Also, get basic data being loaded, and get the fetching to work somewhat ok. Needed to deal w/ the fact that HighStocks does stupid requests data... Issues still present: not timezone aware caps between days in the data not all navigator series are bars --- .gitignore | 1 + Makefile | 24 ++++++- NOTES.md | 15 ++++ README.md | 10 +++ cmds.txt | 1 + fakedata.py | 133 ++++++++++++++++++++++++++++++++++++ root/js/solardash.base.js | 43 +++++++++++- root/js/solardash.file.jspp | 74 ++++++++++++++------ 8 files changed, 276 insertions(+), 25 deletions(-) create mode 100644 cmds.txt create mode 100644 fakedata.py diff --git a/.gitignore b/.gitignore index 91160fd..32ac01b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ solardash.egg-info p *.pyc .coverage +fakedata.js root/js/solardash.file.js root/js/solardash.https.js root/js/solardash.http.js diff --git a/Makefile b/Makefile index 49f4218..f74cdd1 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,21 @@ FILES=$(PROJNAME)/__init__.py JSFILES = root/js/solardash.file.js root/js/solardash.https.js root/js/solardash.http.js -THIRDPARTYJS = root/js/jquery.js root/js/highstock.js +THIRDPARTYJS = \ + root/js/jquery.js \ + root/js/highstock.js \ + +# root/js/moment.min.js \ +# root/js/moment-timezone-with-data.min.js root/js/jquery.js: - wget -O $@ "https://code.jquery.com/jquery-3.4.1.min.js" + wget -O $@ "https://code.jquery.com/jquery-3.4.1.min.js" || (rm "$@"; false) + +root/js/moment.min.js: + wget -O $@ "https://momentjs.com/downloads/moment.min.js" || (rm "$@"; false) + +root/js/moment-timezone-with-data.min.js: + wget -O $@ "https://momentjs.com/downloads/moment-timezone-with-data-1970-2030.js" || (rm "$@"; false) root/js/highstock.js: Makefile wget -O - "https://code.highcharts.com/stock/8.0.0/highstock.js" | grep -v '^//# sourceMappingURL=' > $@ || (rm "$@"; false) @@ -36,8 +47,15 @@ run: $(JSFILES) $(JSFILES): $(THIRDPARTYJS) $(JSBASE) +root/js/solardash.file.js: fakedata.js + +fakedata.js: fakedata.py + python $< > $@ || (rm $@; false) + .jspp.js: - cat $< $(THIRDPARTYJS) $(JSBASE) > $@ || (rm "$@"; false) + # bsdmake uses $>, gmake uses $^ + (echo '// DO NOT EDIT FILE!!!! THIS IS AUTOMATICALLY GENERATED!!!'; cat $^) > $@ || (rm "$@"; false) + #cat $< $(THIRDPARTYJS) $(JSBASE) > $@ || (rm $@; false) keepupdate: find . -name '*.js' -o -name '*.jspp' | entr make all diff --git a/NOTES.md b/NOTES.md index f2cec46..58fd784 100644 --- a/NOTES.md +++ b/NOTES.md @@ -52,6 +52,14 @@ Maybe heatmap for solar panels Use removePoint and addPoint: https://web.archive.org/web/20200109083308/https://api.highcharts.com/class-reference/Highcharts.Series.html to add/remove the end null point and add points as they dynamically arrive +Sample Home power Consumption dataset: +https://archive.ics.uci.edu/ml/datasets/individual+household+electric+power+consumption + +List of Solar resources: +https://energydemo.github.io/SolarDatasets/ + +https://pvoutput.org + WebSocket Definition -------------------- @@ -73,6 +81,13 @@ Messages from the websocket: ov overview of data, contains gird, production and consumption data. following is JSON object + win The graph has set a window size. The two arguments are start and end. If this is the + first time, NaN will be sent for both, which means to send all data. + + windata Data in response to a window command. The format is an object w/ the keys, production, + grid and consumption. Each will be a list of data points. Each point is a pair of + timestamp in miliseconds and the value for the point. + One the first connection, the following messages/data will be sent: c, ng, p, ov diff --git a/README.md b/README.md index 244fb35..e1daccd 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,13 @@ Solar Dashboard =============== This is a solar dashboard for displaying information about a solar install. + +Structure +--------- + +Frontend + +The front end is located in root. The meat of the logic is in +`js/solardash.base.js`. The testing infrastructure is in +`js/solardash.file.js` as if you are launching the page from a file +url, there is no way to make a url to access the backend. diff --git a/cmds.txt b/cmds.txt new file mode 100644 index 0000000..873b282 --- /dev/null +++ b/cmds.txt @@ -0,0 +1 @@ +ls fakedata.py | entr sh -c 'python fakedata.py > fake.txt; echo plot \"fake.txt\" | gnuplot' diff --git a/fakedata.py b/fakedata.py new file mode 100644 index 0000000..428dcde --- /dev/null +++ b/fakedata.py @@ -0,0 +1,133 @@ +import arrow +import random +import pprint +import json + +from datetime import datetime, timedelta + +rand = random.Random('a seed') + +tz = 'US/Pacific' + +startdate = arrow.Arrow(2019, 11, 1, tzinfo=tz) +enddate = arrow.Arrow(2019, 12, 10, tzinfo=tz) + +meanwhprod = 20000 +sigwhprod = 2000 + +def drange(s, e, interval): + cmpfun = lambda s, e, ts, te: te < e + if e < s: + interval = -interval + cmpfun = lambda s, e, ts, te: te > e + ts = s.clone() + te = ts + interval + #print('dr:', repr((s, e, ts, te, cmpfun(s, e, ts, te), interval))) + while cmpfun(s, e, ts, te): + yield ts + (te - ts) + ts, te = te, te + interval + +# idea: +# first hour linear ramp up (25% in first half, 75% in second half) +# middle 5 hours near constant generation +# first/tailing linear is equiv of an hour total, so total power / 6 + +# approx 7 hours time +def makestartend(t): + s = t.replace(hour=9).shift(minutes=rand.gauss(30, 10)) + e = t.replace(hour=16).shift(minutes=rand.gauss(30, 10)) + + return (s, e) + +def normdist(small, big, amount, minsize): + '''Distribute most twoards the big side, total distribute amount + over the entire range, [small, big]. + ''' + + ret = [] + + timediff = abs(big - small) + if timediff < minsize: + scaledamount = amount * (timedelta(hours=1) / timediff) + midpnt = small + (big - small) / 2 + #print('ndf:', repr((small, big, midpnt, amount, scaledamount))) + return [ (midpnt, scaledamount) ] + + #print('nd:', repr((small, big, amount))) + dist = big - small + halfpoint = small + (dist / 9 * 5) + ret.extend(normdist(small, halfpoint, amount / 2, minsize)) + ret.extend(normdist(halfpoint, big, amount / 2, minsize)) + + #print('ndr:') + #pprint.pprint(ret) + return ret + +def linramp(start, end, wtarget, minsize): + mid = start + (end - start) / 2 + timediff = abs(end - start) + ret = [] + + ndates = abs((end - start) / minsize) + #print('lr', ndates) + for x, i in enumerate(drange(start, end, minsize)): + #print(repr((x, i))) + yield (i, (x + 1) / ndates * wtarget) + +def distribute(s, e, prod, minsize): + onehour = timedelta(hours=1) + totaltime = e - s + mid = s + totaltime / 2 + startrampend = s + onehour + endrampstart = e - onehour + + # prod == wh + wtarget = prod / ((totaltime + onehour).seconds / 60 / 60) + + ret = [] + + #print('d:', repr((s, e))) + ret.extend(linramp(s, startrampend, wtarget, minsize)) + + for i in drange(startrampend, endrampstart, minsize): + ret.append((i, wtarget)) + + ret.extend(linramp(e, endrampstart, wtarget, minsize)) + + ret.sort() + #pprint.pprint(ret) + + return ret + +#print('start') + +points = [] +index = [] + +def serializearrowasmili(obj): + if not isinstance(obj, arrow.Arrow): + raise TypeError + + return int(obj.float_timestamp*1000) + +for i in arrow.Arrow.range('day', startdate, enddate): + whprod = rand.gauss(meanwhprod, sigwhprod) + s, e = makestartend(i) + noon = i.replace(hour=12) + + index.append((noon, whprod)) + #print(repr(i), whprod) + dist = distribute(s, e, whprod, timedelta(seconds=20)) + + # print timestamps as miliseconds + if False: + dist = ((int(a.float_timestamp*1000), b) for a, b in dist) + + # print space delimited time, else json + if False: + print('\n'.join('%s %s' % (a, b) for a, b in dist)) + else: + #print(json.dumps(tuple(dist), indent=2)) + points.extend(dist) + +print('fakedata =', json.dumps(dict(production=points, index=index), default=serializearrowasmili)) diff --git a/root/js/solardash.base.js b/root/js/solardash.base.js index 0220f9d..16af13a 100644 --- a/root/js/solardash.base.js +++ b/root/js/solardash.base.js @@ -1,3 +1,9 @@ +//Highcharts.setOptions({ +// time: { +// timezone: 'America/Los_Angeles' +// } +//}); + socket.onopen = function() { // connected } @@ -20,6 +26,19 @@ function netgridcolor(v) { return "#ff0000"; } +function afterSetExtremes(e) { + // hack to deal w/ HighStocks being stupid + if (Date.now() - solarchart.sd_lastwindata < 500) { + // Make sure we trigger again, and that the user doesn't + // have to wait too long + solarchart.sd_lastwindata = Date.now() - 500; + return; + } + solarchart.showLoading('Loading data from server...'); + console.log(e) + socket.send('win ' + Math.round(e.min).toString() + ' ' + Math.round(e.max).toString()); +} + socket.onmessage = function(m) { var msg = m.data.split(" "); if (msg[0] in msgNumbs) { @@ -40,6 +59,15 @@ socket.onmessage = function(m) { solarchart.navigator.series[gridIdx].setData(data.grid, false); solarchart.navigator.series[prodIdx].setData(data.production, false); solarchart.redraw(); + } else if (msg[0] == 'windata') { + var data = JSON.parse(msg[1]); + console.log(data); + solarchart.sd_lastwindata = Date.now(); + solarchart.series[consumIdx].setData(data.consumption, false); + //solarchart.navigator.series[gridIdx].setData(data.grid, false); + //solarchart.navigator.series[prodIdx].setData(data.production, false); + solarchart.redraw(); + solarchart.hideLoading() } } @@ -59,16 +87,19 @@ var solarchart = Highcharts.stockChart('solarchart', { adaptToUpdatedData: false, // XXX - keep? series: [ { + type: 'bar', name: 'consumptionNav', showInNavigator: true, data: [], }, { + type: 'bar', name: 'gridNav', showInNavigator: true, data: [], }, { + type: 'bar', name: 'productionNav', showInNavigator: true, data: [], @@ -76,6 +107,9 @@ var solarchart = Highcharts.stockChart('solarchart', { ] }, xAxis: { + events: { + afterSetExtremes: afterSetExtremes, + }, type: "datetime", title: { enabled: false @@ -83,12 +117,12 @@ var solarchart = Highcharts.stockChart('solarchart', { }, yAxis: { title: { - text: 'kW' + text: 'W' } }, tooltip: { split: true, - valueSuffix: ' kW' + valueSuffix: ' W' }, plotOptions: { area: { @@ -104,7 +138,10 @@ var solarchart = Highcharts.stockChart('solarchart', { series: [ { name: 'grid', - data: fakeData, + type: 'area', + gapSize: 1, + //gapUnit: 'value', + data: [ ], }, ] }); diff --git a/root/js/solardash.file.jspp b/root/js/solardash.file.jspp index e74936c..f382e5b 100644 --- a/root/js/solardash.file.jspp +++ b/root/js/solardash.file.jspp @@ -7,7 +7,45 @@ function WebSocketTest(actions) { this.processNextItem() } -// WebSocketTest.prototype.send = function () +Array.prototype.bisect = function (val, lo, hi) { + var mid; + if (lo == null) + lo = 0; + if (hi == null) + hi = this.length; + + while (lo < hi) { + mid = Math.floor((lo + hi) / 2); + if (this[mid] < val) + lo = mid + 1; + else + hi = mid; + } + return lo; +}; + +WebSocketTest.prototype.send = function (s) { + console.log("ws send: " + s); + var msg = s.split(" "); + + if (msg[0] == 'win') { + if (msg[1] == 'NaN') + msg[1] = -Infinity; + if (msg[2] == 'NaN') + msg[2] = Infinity; + var loidx, hiidx; + loidx = fakedata.production.bisect([msg[1], 0]); + hiidx = fakedata.production.bisect([msg[2], 0]); + var subar = fakedata.production.slice(loidx, hiidx + 1); + var data = { + "production": subar, + "consumption": subar, + "grid": subar, + } + this.makercv('windata ' + JSON.stringify(data)); + } +} + // WebSocketTest.prototype.close = function () // Internal @@ -26,6 +64,9 @@ WebSocketTest.prototype.processNextItem = function () { return this; } +WebSocketTest.prototype.makercv = function (m) { + this.onmessage(new MessageEvent('websockettest', { data: m })); +} fakeData = [ [ 1578544199000, 0.3934 ], @@ -59,26 +100,21 @@ fakeOverviewData = { production: fakeConData, } +function getoverviewdata() { + return { + grid: fakedata.index, + consumption: fakedata.index, + production: fakedata.index, + } +} // Setup the socket that will be used var socket = new WebSocketTest([ [ 10, function(a) { a.onopen(new Object()) } ], - [ 10, function(a) { a.onmessage(new MessageEvent('websockettest', { - data : 'o ' + JSON.stringify(fakeOverviewData) - })) } ], - [ 10, function(a) { a.onmessage(new MessageEvent('websockettest', { - data : 'p .123' - })) } ], - [ 10, function(a) { a.onmessage(new MessageEvent('websockettest', { - data : 'c .302' - })) } ], - [ 10, function(a) { a.onmessage(new MessageEvent('websockettest', { - data : 'ng .758' - })) } ], - [ 2000, function(a) { a.onmessage(new MessageEvent('websockettest', { - data : 'p .234' - })) } ], - [ 10, function(a) { a.onmessage(new MessageEvent('websockettest', { - data : 'ng -.584' - })) } ], + [ 10, function(a) { a.makercv('o ' + JSON.stringify(getoverviewdata())) } ], + [ 10, function(a) { a.makercv('p .123') } ], + [ 10, function(a) { a.makercv('c .302') } ], + [ 10, function(a) { a.makercv('ng .758') } ], + [ 2000, function(a) { a.makercv('p .234') } ], + [ 10, function(a) { a.makercv('ng -.584') } ], ]);