Commit 81984a30 authored by Vladislav Rykov's avatar Vladislav Rykov
Browse files

msgpack data model integrated and tested

parent 45a15aab
...@@ -2,7 +2,6 @@ from psycopg2 import sql ...@@ -2,7 +2,6 @@ from psycopg2 import sql
from app.helpers.misc import with_psql from app.helpers.misc import with_psql
from datetime import datetime from datetime import datetime
@with_psql @with_psql
def create_table(cur, appkey, devid): def create_table(cur, appkey, devid):
tn = 'dev_' +str(appkey)+ '_' +str(devid) tn = 'dev_' +str(appkey)+ '_' +str(devid)
......
...@@ -2,6 +2,9 @@ import msgpack ...@@ -2,6 +2,9 @@ import msgpack
import struct import struct
from collections import OrderedDict from collections import OrderedDict
import json import json
import app.helpers.misc as misc
from psycopg2 import Binary
from datetime import datetime
MODELS = { MODELS = {
'json' : 'JSON', 'json' : 'JSON',
...@@ -71,20 +74,6 @@ def test_dev(): ...@@ -71,20 +74,6 @@ def test_dev():
print ('raw data json:') print ('raw data json:')
print (dru) print (dru)
def read_data_ddm(data, ddm):
if ddm['model'] == 'mpack':
return msgpack.unpackb(data)
elif ddm['model'] == 'raw':
upstr = ddm['endianness'] + ''.join(dict(ddm['format']).values())
data = struct.unpack(upstr, data)
data = dict(zip(ddm['format'].keys(), data))
for k, v in ddm['format'].items():
if v[-1] == 's':
data[k] = data[k].decode('utf-8')
return data
elif ddm['model'] == 'json':
return json.loads(data.decode('utf-8'))
def test_done(): def test_done():
print ('test with json:') print ('test with json:')
ddm = { ddm = {
...@@ -105,3 +94,73 @@ def test_done(): ...@@ -105,3 +94,73 @@ def test_done():
'format' : OrderedDict([('some_int', 'H'), ('some_float', 'f'), ('some_bool', '?'), ('some_str', '7s')]) 'format' : OrderedDict([('some_int', 'H'), ('some_float', 'f'), ('some_bool', '?'), ('some_str', '7s')])
} }
print(read_data_ddm(rdata, ddm)) print(read_data_ddm(rdata, ddm))
@misc.with_psql
def insert_test(cur):
import random
m = {
"temperature" : random.randint(0, 100),
"lever": random.randint(0,1)
}
m = msgpack.packb(m)
query = """
INSERT INTO dev_3b56f3d8_3 VALUES ({}, '{}', {})
""".format(misc.get_utc(), datetime.now().strftime('%H:%M:%S'), Binary(m))
print (query)
cur.execute(query)
return (True,)
def decode_datum(data, ddm):
data = [d for d in data]
data[2] = read_data(data[2].tobytes(), ddm)
return data
def read_data(data, ddm):
if ddm['model'] == 'mpack':
return msgpack.unpackb(data)
elif ddm['model'] == 'raw':
upstr = ddm['endianness'] + ''.join(dict(ddm['format']).values())
data = struct.unpack(upstr, data)
data = dict(zip(ddm['format'].keys(), data))
for k, v in ddm['format'].items():
if v[-1] == 's':
data[k] = data[k].decode('utf-8')
return data
elif ddm['model'] == 'json':
return json.loads(data.decode('utf-8'))
def extract(request):
ddmin = {'model':request.form['ddm'], 'format':{}}
try:
ddmin['endianness'] = request.form['endianness']
except:
pass
# create dict with variables
for k,v in request.form.items():
if k.startswith("var"):
i = k.split("_")
if not int(i[1]) in ddmin['format']:
ddmin['format'][int(i[1])] = { i[0][3:] : v }
else:
ddmin['format'][int(i[1])][i[0][3:]] = v
# format size
for k,v in ddmin['format'].items():
if 'size' in v:
ddmin['format'][k]['type'] = v['size'] + 's'
ddmin['format'][k].pop('size')
# order dict
od = collections.OrderedDict(sorted(ddmin['format'].items()))
ddmin.pop('format')
ddmin['format'] = collections.OrderedDict()
# give it defined ddm format
for k,v in od.items():
ddmin['format'][v['name']] = v['type']
return ddmin
print('insert', insert_test())
...@@ -173,32 +173,5 @@ def local_weekday(day_offset = 0): ...@@ -173,32 +173,5 @@ def local_weekday(day_offset = 0):
def utc_local_diff(): def utc_local_diff():
return abs((datetime.now() - datetime.utcnow()).total_seconds()) return abs((datetime.now() - datetime.utcnow()).total_seconds())
def extract_ddm(request): def get_utc():
ddmin = {'model':request.form['ddm'], 'format':{}} return int((datetime.now() - datetime(1970,1,1)).total_seconds())
try:
ddmin['endianness'] = request.form['endianness']
except:
pass
# create dict with variables
for k,v in request.form.items():
if k.startswith("var"):
i = k.split("_")
if not int(i[1]) in ddmin['format']:
ddmin['format'][int(i[1])] = { i[0][3:] : v }
else:
ddmin['format'][int(i[1])][i[0][3:]] = v
# format size
for k,v in ddmin['format'].items():
if 'size' in v:
ddmin['format'][k]['type'] = v['size'] + 's'
ddmin['format'][k].pop('size')
# order dict
od = collections.OrderedDict(sorted(ddmin['format'].items()))
ddmin.pop('format')
ddmin['format'] = collections.OrderedDict()
# give it defined ddm format
for k,v in od.items():
ddmin['format'][v['name']] = v['type']
return ddmin
...@@ -90,10 +90,10 @@ ...@@ -90,10 +90,10 @@
</div> </div>
</div> </div>
</div> </div>
{% if data %} {% if total > 0 %}
<div> <div>
<ul class="nav nav-pills" id="pills-tab" role="tablist"> <ul class="nav nav-pills" id="pills-tab" role="tablist">
{% for k in data[0][2] %} {% for k in dev[3]['format'] %}
<li class="nav-item col-lg-3 col-md-6" role="presentation"> <li class="nav-item col-lg-3 col-md-6" role="presentation">
<a class="nav-link" id="tab_{{ k }}" data-toggle="pill" href="#{{ k }}" role="tab" aria-controls="{{ k }}" aria-selected="true" onclick="display_data('{{ k }}')">{{ k }}</a> <a class="nav-link" id="tab_{{ k }}" data-toggle="pill" href="#{{ k }}" role="tab" aria-controls="{{ k }}" aria-selected="true" onclick="display_data('{{ k }}')">{{ k }}</a>
</li> </li>
...@@ -101,7 +101,7 @@ ...@@ -101,7 +101,7 @@
</ul> </ul>
<div class="tab-content" id="pills-tabContent"> <div class="tab-content" id="pills-tabContent">
{% for k in data[0][2] %} {% for k in dev[3]['format'] %}
<div class="tab-pane fade card" id="{{ k }}" style="margin-top: 30px;" role="tabpanel" aria-labelledby="tab_{{ k }}"> <div class="tab-pane fade card" id="{{ k }}" style="margin-top: 30px;" role="tabpanel" aria-labelledby="tab_{{ k }}">
<div class="card-body"> <div class="card-body">
<h3 class="mb-0 card-header">Last 24 hours data</h3> <h3 class="mb-0 card-header">Last 24 hours data</h3>
...@@ -117,7 +117,9 @@ ...@@ -117,7 +117,9 @@
<tbody id="table_{{ k }}_body"> <tbody id="table_{{ k }}_body">
</tbody> </tbody>
</table> </table>
<center><a href="javascript:void(0);" onclick="return table_load_more('{{ k }}');">Load more</a></center> {% if total > table_max %}
<center><a href="javascript:void(0);" id="table_load_more" onclick="return table_load_more('{{ k }}');">Load more</a></center>
{% endif %}
</div> </div>
</div> </div>
{% endfor %} {% endfor %}
...@@ -149,7 +151,7 @@ ...@@ -149,7 +151,7 @@
{% endblock %} {% endblock %}
{% block script %} {% block script %}
{% if data %} {% if total > 0 %}
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.0/jquery.min.js"></script> <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.0/jquery.min.js"></script>
<script type="text/javascript"> <script type="text/javascript">
google.charts.load('current', {'packages':['corechart']}); google.charts.load('current', {'packages':['corechart']});
...@@ -157,7 +159,7 @@ ...@@ -157,7 +159,7 @@
function init_scroll() { function init_scroll() {
var ts = { var ts = {
{% for k in data[0][2] %} {% for k in dev[3]['format'] %}
{{ k }} : 1, {{ k }} : 1,
{% endfor %} {% endfor %}
}; };
...@@ -203,13 +205,20 @@ ...@@ -203,13 +205,20 @@
} }
function show_first_page() { function show_first_page() {
document.getElementById("tab_{{ data[0][2] | first }}").click(); document.getElementById("tab_{{ dev[3]['format'] | first }}").click();
} }
function table_load_more(dname) { function table_load_more(dname) {
tscroll[dname] += 1; tscroll[dname] += 1;
fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(data => $('#table_'+dname+'_body').append(data)); fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(function (data)
fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(data => console.log(data)); {
if (data.length > 0) {
$('#table_'+dname+'_body').append(data);
} else {
$("#table_load_more").remove();
}
}
);
} }
</script> </script>
{% endif %} {% endif %}
......
...@@ -200,9 +200,9 @@ def application_device(appkey, devid): ...@@ -200,9 +200,9 @@ def application_device(appkey, devid):
ltup = ld[1][0][1] ltup = ld[1][0][1]
if ld[0]: if ld[0]:
return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, data=ld[1], total=cnt[1][0]) return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, total=cnt[1][0], table_max=MAX_PG_ENTRIES_DATA)
else: else:
return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, data=[], total=cnt[1][0]) return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, total=cnt[1][0])
else: else:
return redirect(url_for('login')) return redirect(url_for('login'))
...@@ -214,24 +214,8 @@ def application_add_device(appkey): ...@@ -214,24 +214,8 @@ def application_add_device(appkey):
dev_list = dd.get_list(appkey) dev_list = dd.get_list(appkey)
return render_template('new/public/add-device.html', app=ap[1], free_ids=misc.prep_id_range(dev_list[1]), models=ddm.MODELS) return render_template('new/public/add-device.html', app=ap[1], free_ids=misc.prep_id_range(dev_list[1]), models=ddm.MODELS)
elif request.method == 'POST': elif request.method == 'POST':
ddmin = misc.extract_ddm(request) ddmin = ddm.extract(request)
if False:
res = dd.create(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'])
if not res[0]:
flash('Error: {}'.format(res[1]), 'danger')
return render_template(request.url)
else:
res = data.create_table(appkey, request.form['devid'])
if not res[0]:
dd.delete(session['appkey'], request.form['devid'])
flash('Error: {}'.format(res[1]), 'danger')
return render_template(request.url)
else:
return redirect(url_for('application', appkey=appkey))
if True:
res = dd.create_ddm(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'], ddmin) res = dd.create_ddm(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'], ddmin)
if not res[0]: if not res[0]:
flash('Error: {}'.format(res[1]), 'danger') flash('Error: {}'.format(res[1]), 'danger')
...@@ -443,40 +427,16 @@ def settings(): ...@@ -443,40 +427,16 @@ def settings():
return redirect(request.url) return redirect(request.url)
@app.route('/dev-data/<var>/<dest>/<page>')
def dev_data(var, dest, page):
if dest == 'graph':
last = data.get_last_hours(session['appkey'], session['devid'], MAX_PG_ENTRIES_GRAPH_HOURS, int(page))
arr = '[["Time", "{}"],'.format(var)
if last[0]:
for d in last[1]:
arr += '[new Date('+str(d[0])+'*1000),'+str(d[2][var])+'],'
arr += ']'
return arr
elif dest == 'table':
# for table <cnt> is in items
last = data.get_last_range(session['appkey'], session['devid'], [MAX_PG_ENTRIES_DATA, (int(page)-1)*MAX_PG_ENTRIES_DATA])
#t = """ <thead>
# <th>Time</th>
# <th>{}</th>
# </thead>
# <tbody>
#""".format(var)
t = ''
if last[0]:
for d in last[1]:
t += '<tr><th>'+d[1]+'</th><th>'+str(d[2][var])+'</th></tr>'
#t += '</tbody>'
return t
@app.route('/application/<appkey>/device/<devid>/data/<var>/<dest>/<page>') @app.route('/application/<appkey>/device/<devid>/data/<var>/<dest>/<page>')
def application_device_data(appkey, devid, var, dest, page): def application_device_data(appkey, devid, var, dest, page):
dev = dd.get(appkey, devid)[1]
if dest == 'graph': if dest == 'graph':
last = data.get_last_hours(appkey, devid, MAX_PG_ENTRIES_GRAPH_HOURS, int(page)) last = data.get_last_hours(appkey, devid, MAX_PG_ENTRIES_GRAPH_HOURS, int(page))
arr = '[["Time", "{}"],'.format(var) arr = ''
if last[0]: if last[0]:
for d in last[1]: arr = '[["Time", "{}"],'.format(var)
last = [ddm.decode_datum(d, dev[3]) for d in last[1]]
for d in last:
arr += '[new Date('+str(d[0])+'*1000),'+str(d[2][var])+'],' arr += '[new Date('+str(d[0])+'*1000),'+str(d[2][var])+'],'
arr += ']' arr += ']'
return arr return arr
...@@ -485,7 +445,8 @@ def application_device_data(appkey, devid, var, dest, page): ...@@ -485,7 +445,8 @@ def application_device_data(appkey, devid, var, dest, page):
last = data.get_last_range(appkey, devid, [MAX_PG_ENTRIES_DATA, (int(page)-1)*MAX_PG_ENTRIES_DATA]) last = data.get_last_range(appkey, devid, [MAX_PG_ENTRIES_DATA, (int(page)-1)*MAX_PG_ENTRIES_DATA])
t = '' t = ''
if last[0]: if last[0]:
for d in last[1]: last = [ddm.decode_datum(d, dev[3]) for d in last[1]]
for d in last:
t += '<tr><th>'+d[1]+'</th><th>'+str(d[2][var])+'</th></tr>' t += '<tr><th>'+d[1]+'</th><th>'+str(d[2][var])+'</th></tr>'
return t return t
...@@ -629,8 +590,7 @@ def application_device_settings(appkey, devid): ...@@ -629,8 +590,7 @@ def application_device_settings(appkey, devid):
return render_template('new/public/device-settings.html', app=ap[1], dev=dev[1], models=ddm.MODELS) return render_template('new/public/device-settings.html', app=ap[1], dev=dev[1], models=ddm.MODELS)
elif request.method == 'POST': elif request.method == 'POST':
ddmin = misc.extract_ddm(request) ddmin = ddm.extract(request)
#res = dd.update(appkey, devid, request.form['devname'], request.form['devdesc'])
res = dd.update_ddm(appkey, devid, request.form['devname'], request.form['devdesc'], ddmin) res = dd.update_ddm(appkey, devid, request.form['devname'], request.form['devdesc'], ddmin)
if not res[0]: if not res[0]:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment