Skip to content

GitLab

Commit 81984a30 authored by Vladislav Rykov's avatar Vladislav Rykov
Browse files

msgpack data model integrated and tested

parent 45a15aab
Show whitespace changes
Inline Side-by-side
app/app/dao/data/data.py
View file @ 81984a30
......@@ -2,7 +2,6 @@ from psycopg2 import sql
from app.helpers.misc import with_psql
from datetime import datetime
@with_psql
def create_table(cur, appkey, devid):
tn = 'dev_' +str(appkey)+ '_' +str(devid)
......
app/app/helpers/device_data_model.py
View file @ 81984a30
......@@ -2,6 +2,9 @@ import msgpack
import struct
from collections import OrderedDict
import json
import app.helpers.misc as misc
from psycopg2 import Binary
from datetime import datetime
MODELS = {
'json' : 'JSON',
......@@ -71,20 +74,6 @@ def test_dev():
print ('raw data json:')
print (dru)
def read_data_ddm(data, ddm):
if ddm['model'] == 'mpack':
return msgpack.unpackb(data)
elif ddm['model'] == 'raw':
upstr = ddm['endianness'] + ''.join(dict(ddm['format']).values())
data = struct.unpack(upstr, data)
data = dict(zip(ddm['format'].keys(), data))
for k, v in ddm['format'].items():
if v[-1] == 's':
data[k] = data[k].decode('utf-8')
return data
elif ddm['model'] == 'json':
return json.loads(data.decode('utf-8'))
def test_done():
print ('test with json:')
ddm = {
......@@ -105,3 +94,73 @@ def test_done():
'format' : OrderedDict([('some_int', 'H'), ('some_float', 'f'), ('some_bool', '?'), ('some_str', '7s')])
}
print(read_data_ddm(rdata, ddm))
@misc.with_psql
def insert_test(cur):
import random
m = {
"temperature" : random.randint(0, 100),
"lever": random.randint(0,1)
}
m = msgpack.packb(m)
query = """
INSERT INTO dev_3b56f3d8_3 VALUES ({}, '{}', {})
""".format(misc.get_utc(), datetime.now().strftime('%H:%M:%S'), Binary(m))
print (query)
cur.execute(query)
return (True,)
def decode_datum(data, ddm):
data = [d for d in data]
data[2] = read_data(data[2].tobytes(), ddm)
return data
def read_data(data, ddm):
if ddm['model'] == 'mpack':
return msgpack.unpackb(data)
elif ddm['model'] == 'raw':
upstr = ddm['endianness'] + ''.join(dict(ddm['format']).values())
data = struct.unpack(upstr, data)
data = dict(zip(ddm['format'].keys(), data))
for k, v in ddm['format'].items():
if v[-1] == 's':
data[k] = data[k].decode('utf-8')
return data
elif ddm['model'] == 'json':
return json.loads(data.decode('utf-8'))
def extract(request):
ddmin = {'model':request.form['ddm'], 'format':{}}
try:
ddmin['endianness'] = request.form['endianness']
except:
pass
# create dict with variables
for k,v in request.form.items():
if k.startswith("var"):
i = k.split("_")
if not int(i[1]) in ddmin['format']:
ddmin['format'][int(i[1])] = { i[0][3:] : v }
else:
ddmin['format'][int(i[1])][i[0][3:]] = v
# format size
for k,v in ddmin['format'].items():
if 'size' in v:
ddmin['format'][k]['type'] = v['size'] + 's'
ddmin['format'][k].pop('size')
# order dict
od = collections.OrderedDict(sorted(ddmin['format'].items()))
ddmin.pop('format')
ddmin['format'] = collections.OrderedDict()
# give it defined ddm format
for k,v in od.items():
ddmin['format'][v['name']] = v['type']
return ddmin
print('insert', insert_test())
app/app/helpers/misc.py
View file @ 81984a30
......@@ -173,32 +173,5 @@ def local_weekday(day_offset = 0):
def utc_local_diff():
return abs((datetime.now() - datetime.utcnow()).total_seconds())
def extract_ddm(request):
ddmin = {'model':request.form['ddm'], 'format':{}}
try:
ddmin['endianness'] = request.form['endianness']
except:
pass
# create dict with variables
for k,v in request.form.items():
if k.startswith("var"):
i = k.split("_")
if not int(i[1]) in ddmin['format']:
ddmin['format'][int(i[1])] = { i[0][3:] : v }
else:
ddmin['format'][int(i[1])][i[0][3:]] = v
# format size
for k,v in ddmin['format'].items():
if 'size' in v:
ddmin['format'][k]['type'] = v['size'] + 's'
ddmin['format'][k].pop('size')
# order dict
od = collections.OrderedDict(sorted(ddmin['format'].items()))
ddmin.pop('format')
ddmin['format'] = collections.OrderedDict()
# give it defined ddm format
for k,v in od.items():
ddmin['format'][v['name']] = v['type']
return ddmin
def get_utc():
return int((datetime.now() - datetime(1970,1,1)).total_seconds())
app/app/templates/new/public/device.html
View file @ 81984a30
......@@ -90,10 +90,10 @@
</div>
</div>
</div>
{% if data %}
{% if total > 0 %}
<div>
<ul class="nav nav-pills" id="pills-tab" role="tablist">
{% for k in data[0][2] %}
{% for k in dev[3]['format'] %}
<li class="nav-item col-lg-3 col-md-6" role="presentation">
<a class="nav-link" id="tab_{{ k }}" data-toggle="pill" href="#{{ k }}" role="tab" aria-controls="{{ k }}" aria-selected="true" onclick="display_data('{{ k }}')">{{ k }}</a>
</li>
......@@ -101,7 +101,7 @@
</ul>
<div class="tab-content" id="pills-tabContent">
{% for k in data[0][2] %}
{% for k in dev[3]['format'] %}
<div class="tab-pane fade card" id="{{ k }}" style="margin-top: 30px;" role="tabpanel" aria-labelledby="tab_{{ k }}">
<div class="card-body">
<h3 class="mb-0 card-header">Last 24 hours data</h3>
......@@ -117,7 +117,9 @@
<tbody id="table_{{ k }}_body">
</tbody>
</table>
<center><a href="javascript:void(0);" onclick="return table_load_more('{{ k }}');">Load more</a></center>
{% if total > table_max %}
<center><a href="javascript:void(0);" id="table_load_more" onclick="return table_load_more('{{ k }}');">Load more</a></center>
{% endif %}
</div>
</div>
{% endfor %}
......@@ -149,7 +151,7 @@
{% endblock %}
{% block script %}
{% if data %}
{% if total > 0 %}
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.0/jquery.min.js"></script>
<script type="text/javascript">
google.charts.load('current', {'packages':['corechart']});
......@@ -157,7 +159,7 @@
function init_scroll() {
var ts = {
{% for k in data[0][2] %}
{% for k in dev[3]['format'] %}
{{ k }} : 1,
{% endfor %}
};
......@@ -203,13 +205,20 @@
}
function show_first_page() {
document.getElementById("tab_{{ data[0][2] | first }}").click();
document.getElementById("tab_{{ dev[3]['format'] | first }}").click();
}
function table_load_more(dname) {
tscroll[dname] += 1;
fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(data => $('#table_'+dname+'_body').append(data));
fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(data => console.log(data));
fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(function (data)
{
if (data.length > 0) {
$('#table_'+dname+'_body').append(data);
} else {
$("#table_load_more").remove();
}
}
);
}
</script>
{% endif %}
......
app/app/views.py
View file @ 81984a30
......@@ -200,9 +200,9 @@ def application_device(appkey, devid):
ltup = ld[1][0][1]
if ld[0]:
return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, data=ld[1], total=cnt[1][0])
return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, total=cnt[1][0], table_max=MAX_PG_ENTRIES_DATA)
else:
return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, data=[], total=cnt[1][0])
return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, total=cnt[1][0])
else:
return redirect(url_for('login'))
......@@ -214,24 +214,8 @@ def application_add_device(appkey):
dev_list = dd.get_list(appkey)
return render_template('new/public/add-device.html', app=ap[1], free_ids=misc.prep_id_range(dev_list[1]), models=ddm.MODELS)
elif request.method == 'POST':
ddmin = misc.extract_ddm(request)
ddmin = ddm.extract(request)
if False:
res = dd.create(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'])
if not res[0]:
flash('Error: {}'.format(res[1]), 'danger')
return render_template(request.url)
else:
res = data.create_table(appkey, request.form['devid'])
if not res[0]:
dd.delete(session['appkey'], request.form['devid'])
flash('Error: {}'.format(res[1]), 'danger')
return render_template(request.url)
else:
return redirect(url_for('application', appkey=appkey))
if True:
res = dd.create_ddm(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'], ddmin)
if not res[0]:
flash('Error: {}'.format(res[1]), 'danger')
......@@ -443,40 +427,16 @@ def settings():
return redirect(request.url)
@app.route('/dev-data/<var>/<dest>/<page>')
def dev_data(var, dest, page):
if dest == 'graph':
last = data.get_last_hours(session['appkey'], session['devid'], MAX_PG_ENTRIES_GRAPH_HOURS, int(page))
arr = '[["Time", "{}"],'.format(var)
if last[0]:
for d in last[1]:
arr += '[new Date('+str(d[0])+'*1000),'+str(d[2][var])+'],'
arr += ']'
return arr
elif dest == 'table':
# for table <cnt> is in items
last = data.get_last_range(session['appkey'], session['devid'], [MAX_PG_ENTRIES_DATA, (int(page)-1)*MAX_PG_ENTRIES_DATA])
#t = """ <thead>
# <th>Time</th>
# <th>{}</th>
# </thead>
# <tbody>
#""".format(var)
t = ''
if last[0]:
for d in last[1]:
t += '<tr><th>'+d[1]+'</th><th>'+str(d[2][var])+'</th></tr>'
#t += '</tbody>'
return t
@app.route('/application/<appkey>/device/<devid>/data/<var>/<dest>/<page>')
def application_device_data(appkey, devid, var, dest, page):
dev = dd.get(appkey, devid)[1]
if dest == 'graph':
last = data.get_last_hours(appkey, devid, MAX_PG_ENTRIES_GRAPH_HOURS, int(page))
arr = '[["Time", "{}"],'.format(var)
arr = ''
if last[0]:
for d in last[1]:
arr = '[["Time", "{}"],'.format(var)
last = [ddm.decode_datum(d, dev[3]) for d in last[1]]
for d in last:
arr += '[new Date('+str(d[0])+'*1000),'+str(d[2][var])+'],'
arr += ']'
return arr
......@@ -485,7 +445,8 @@ def application_device_data(appkey, devid, var, dest, page):
last = data.get_last_range(appkey, devid, [MAX_PG_ENTRIES_DATA, (int(page)-1)*MAX_PG_ENTRIES_DATA])
t = ''
if last[0]:
for d in last[1]:
last = [ddm.decode_datum(d, dev[3]) for d in last[1]]
for d in last:
t += '<tr><th>'+d[1]+'</th><th>'+str(d[2][var])+'</th></tr>'
return t
......@@ -629,8 +590,7 @@ def application_device_settings(appkey, devid):
return render_template('new/public/device-settings.html', app=ap[1], dev=dev[1], models=ddm.MODELS)
elif request.method == 'POST':
ddmin = misc.extract_ddm(request)
#res = dd.update(appkey, devid, request.form['devname'], request.form['devdesc'])
ddmin = ddm.extract(request)
res = dd.update_ddm(appkey, devid, request.form['devname'], request.form['devdesc'], ddmin)
if not res[0]:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment