msgpack data model integrated and tested

app/app/dao/data/data.py

@@ -2,7 +2,6 @@ from psycopg2 import sql
 from app.helpers.misc import with_psql
 from datetime import datetime
 
-
 @with_psql
 def create_table(cur, appkey, devid):
     tn = 'dev_' +str(appkey)+ '_' +str(devid)
@@ -97,7 +96,7 @@ def get_last_hours(cur, appkey, devid, hours, p):
     cur.execute(
         sql.SQL(query).format(sql.Identifier(tn)), [utcb, utcu])
     data = cur.fetchall()
-        
+    
     if (data == []):
         return (False, 'There is no data for the device.')
     else:

app/app/helpers/device_data_model.py

@@ -2,6 +2,9 @@ import msgpack
 import struct
 from collections import OrderedDict
 import json
+import app.helpers.misc as misc
+from psycopg2 import Binary
+from datetime import datetime
 
 MODELS = {
     'json' : 'JSON',
@@ -71,20 +74,6 @@ def test_dev():
     print ('raw data json:')
     print (dru)
 
-def read_data_ddm(data, ddm):
-    if ddm['model'] == 'mpack':
-        return msgpack.unpackb(data)
-    elif ddm['model'] == 'raw':
-        upstr = ddm['endianness'] + ''.join(dict(ddm['format']).values())
-        data = struct.unpack(upstr, data)
-        data = dict(zip(ddm['format'].keys(), data))
-        for k, v in ddm['format'].items():
-            if v[-1] == 's':
-                data[k] = data[k].decode('utf-8')
-        return data 
-    elif ddm['model'] == 'json':
-        return json.loads(data.decode('utf-8'))
-
 def test_done():
     print ('test with json:')
     ddm = {
@@ -105,3 +94,73 @@ def test_done():
         'format' : OrderedDict([('some_int', 'H'), ('some_float', 'f'), ('some_bool', '?'), ('some_str', '7s')])
     }
     print(read_data_ddm(rdata, ddm))
+
+@misc.with_psql
+def insert_test(cur):
+    import random
+    
+    m = {
+            "temperature" : random.randint(0, 100),
+            "lever": random.randint(0,1)
+        }
+    m = msgpack.packb(m)
+    query = """
+        INSERT INTO dev_3b56f3d8_3 VALUES ({}, '{}', {})
+    """.format(misc.get_utc(), datetime.now().strftime('%H:%M:%S'), Binary(m))
+    print (query)
+    cur.execute(query)
+    
+    return (True,)
+
+
+def decode_datum(data, ddm):
+    data = [d for d in data]
+    data[2] = read_data(data[2].tobytes(), ddm)
+    return data
+
+def read_data(data, ddm):
+    if ddm['model'] == 'mpack':
+        return msgpack.unpackb(data)
+    elif ddm['model'] == 'raw':
+        upstr = ddm['endianness'] + ''.join(dict(ddm['format']).values())
+        data = struct.unpack(upstr, data)
+        data = dict(zip(ddm['format'].keys(), data))
+        for k, v in ddm['format'].items():
+            if v[-1] == 's':
+                data[k] = data[k].decode('utf-8')
+        return data 
+    elif ddm['model'] == 'json':
+        return json.loads(data.decode('utf-8'))
+
+
+def extract(request):
+    ddmin = {'model':request.form['ddm'], 'format':{}}
+    try:
+        ddmin['endianness'] = request.form['endianness']
+    except:
+        pass
+
+    # create dict with variables
+    for k,v in request.form.items():
+        if k.startswith("var"):
+            i = k.split("_")
+            if not int(i[1]) in ddmin['format']:
+                ddmin['format'][int(i[1])] = { i[0][3:] : v }
+            else:
+                ddmin['format'][int(i[1])][i[0][3:]] = v
+    # format size
+    for k,v in ddmin['format'].items():
+        if 'size' in v:
+            ddmin['format'][k]['type'] = v['size'] + 's'
+            ddmin['format'][k].pop('size')
+    # order dict
+    od = collections.OrderedDict(sorted(ddmin['format'].items()))
+    ddmin.pop('format')
+    ddmin['format'] = collections.OrderedDict()
+    # give it defined ddm format
+    for k,v in od.items():
+        ddmin['format'][v['name']] = v['type']
+
+    return ddmin
+
+print('insert', insert_test())

app/app/helpers/misc.py

@@ -173,32 +173,5 @@ def local_weekday(day_offset = 0):
 def utc_local_diff():
     return abs((datetime.now() - datetime.utcnow()).total_seconds())
 
-def extract_ddm(request):
-    ddmin = {'model':request.form['ddm'], 'format':{}}
-    try:
-        ddmin['endianness'] = request.form['endianness']
-    except:
-        pass
-
-    # create dict with variables
-    for k,v in request.form.items():
-        if k.startswith("var"):
-            i = k.split("_")
-            if not int(i[1]) in ddmin['format']:
-                ddmin['format'][int(i[1])] = { i[0][3:] : v }
-            else:
-                ddmin['format'][int(i[1])][i[0][3:]] = v
-    # format size
-    for k,v in ddmin['format'].items():
-        if 'size' in v:
-            ddmin['format'][k]['type'] = v['size'] + 's'
-            ddmin['format'][k].pop('size')
-    # order dict
-    od = collections.OrderedDict(sorted(ddmin['format'].items()))
-    ddmin.pop('format')
-    ddmin['format'] = collections.OrderedDict()
-    # give it defined ddm format
-    for k,v in od.items():
-        ddmin['format'][v['name']] = v['type']
-
-    return ddmin
+def get_utc():
+    return int((datetime.now() - datetime(1970,1,1)).total_seconds())

app/app/templates/new/public/device.html

@@ -90,10 +90,10 @@
                       </div>
                     </div>
                 </div>
-		{% if data %}
+		{% if total > 0 %}
                 <div>
                   <ul class="nav nav-pills" id="pills-tab" role="tablist">
-		  {% for k in data[0][2] %}
+		  {% for k in dev[3]['format'] %}
                       <li class="nav-item col-lg-3 col-md-6" role="presentation">
 			      <a class="nav-link" id="tab_{{ k }}" data-toggle="pill" href="#{{ k }}" role="tab" aria-controls="{{ k }}" aria-selected="true" onclick="display_data('{{ k }}')">{{ k }}</a>
                       </li>
@@ -101,7 +101,7 @@
                   </ul>
 
                   <div class="tab-content" id="pills-tabContent">
-		      {% for k in data[0][2] %}
+		      {% for k in dev[3]['format'] %}
 		      <div class="tab-pane fade card" id="{{ k }}" style="margin-top: 30px;" role="tabpanel" aria-labelledby="tab_{{ k }}">
 			<div class="card-body">
               		      <h3 class="mb-0 card-header">Last 24 hours data</h3>
@@ -117,8 +117,10 @@
 				      <tbody id="table_{{ k }}_body">
 				      </tbody>
 			      </table>
-			      <center><a href="javascript:void(0);" onclick="return table_load_more('{{ k }}');">Load more</a></center>
-		        </div>
+			      {% if total > table_max %}
+			      <center><a href="javascript:void(0);" id="table_load_more" onclick="return table_load_more('{{ k }}');">Load more</a></center>
+		              {% endif %}
+			</div>
 		      </div>
 		      {% endfor %}
                   </div>
@@ -149,7 +151,7 @@
 {% endblock %}
 
 {% block script %}
-{% if data %}
+{% if total > 0 %}
 <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.0/jquery.min.js"></script>
 <script type="text/javascript">
 	google.charts.load('current', {'packages':['corechart']});
@@ -157,7 +159,7 @@
 
 	function init_scroll() {
 		var ts = {
-			{% for k in data[0][2] %}
+			{% for k in dev[3]['format'] %}
 			{{ k }} : 1,
 			{% endfor %}
 		};
@@ -203,13 +205,20 @@
 	}
 	
 	function show_first_page() {
-		document.getElementById("tab_{{ data[0][2] | first }}").click();
+		document.getElementById("tab_{{ dev[3]['format'] | first }}").click();
 	}
 	
 	function table_load_more(dname) {
 		tscroll[dname] += 1;
-		fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(data => $('#table_'+dname+'_body').append(data));
-		fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(data => console.log(data));
+		fetch('/application/{{ app[1] }}/device/{{ dev[1] }}/data/'+dname+'/table/'+tscroll[dname]).then(res => res.text()).then(function (data) 
+			{ 
+				if (data.length > 0) { 
+					$('#table_'+dname+'_body').append(data);
+				} else { 
+					$("#table_load_more").remove(); 
+				}
+			}
+		);
 	}
 </script>
 {% endif %}

app/app/views.py

@@ -200,9 +200,9 @@ def application_device(appkey, devid):
                 ltup = ld[1][0][1]
 
             if ld[0]: 
-                return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, data=ld[1], total=cnt[1][0])
+                return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, total=cnt[1][0], table_max=MAX_PG_ENTRIES_DATA)
             else:
-                return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, data=[], total=cnt[1][0])
+                return render_template('new/public/device.html', dev=dev[1], app=ap[1], ltup=ltup, total=cnt[1][0])
     else:
         return redirect(url_for('login'))
 
@@ -214,36 +214,20 @@ def application_add_device(appkey):
             dev_list = dd.get_list(appkey)
             return render_template('new/public/add-device.html', app=ap[1], free_ids=misc.prep_id_range(dev_list[1]), models=ddm.MODELS)
         elif request.method == 'POST':
-            ddmin = misc.extract_ddm(request)
+            ddmin = ddm.extract(request)
            
-            if False:
-                res = dd.create(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'])
-
-                if not res[0]:
-                    flash('Error: {}'.format(res[1]), 'danger')
-                    return render_template(request.url)
-                else:
-                    res = data.create_table(appkey, request.form['devid'])
-                
-                    if not res[0]:
-                        dd.delete(session['appkey'], request.form['devid'])
-                        flash('Error: {}'.format(res[1]), 'danger')
-                        return render_template(request.url)
-                    else:
-                        return redirect(url_for('application', appkey=appkey))
-            if True:
-                res = dd.create_ddm(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'], ddmin)
+            res = dd.create_ddm(request.form['devname'], request.form['devid'], appkey, request.form['devdesc'], ddmin)
+            if not res[0]:
+                flash('Error: {}'.format(res[1]), 'danger')
+                return render_template(request.url)
+            else:
+                res = data.create_table_ddm(appkey, request.form['devid'])
                 if not res[0]:
+                    dd.delete(session['appkey'], request.form['devid'])
                     flash('Error: {}'.format(res[1]), 'danger')
                     return render_template(request.url)
                 else:
-                    res = data.create_table_ddm(appkey, request.form['devid'])
-                    if not res[0]:
-                        dd.delete(session['appkey'], request.form['devid'])
-                        flash('Error: {}'.format(res[1]), 'danger')
-                        return render_template(request.url)
-                    else:
-                        return redirect(url_for('application', appkey=appkey))
+                    return redirect(url_for('application', appkey=appkey))
     else:
         return redirect(url_for('login'))
 
@@ -443,40 +427,16 @@ def settings():
         return redirect(request.url)
 
 
-@app.route('/dev-data/<var>/<dest>/<page>')
-def dev_data(var, dest, page):
-    if dest == 'graph':
-        last = data.get_last_hours(session['appkey'], session['devid'], MAX_PG_ENTRIES_GRAPH_HOURS, int(page))
-        arr = '[["Time", "{}"],'.format(var)
-        if last[0]:
-            for d in last[1]:
-                arr += '[new Date('+str(d[0])+'*1000),'+str(d[2][var])+'],'
-            arr += ']'
-        return arr
-    elif dest == 'table':
-        # for table <cnt> is in items
-        last = data.get_last_range(session['appkey'], session['devid'], [MAX_PG_ENTRIES_DATA, (int(page)-1)*MAX_PG_ENTRIES_DATA])
-        #t = """ <thead>
-        #            <th>Time</th>
-        #            <th>{}</th>
-        #        </thead>
-        #        <tbody>
-        #""".format(var)
-        t = ''
-        if last[0]:
-            for d in last[1]:
-                t += '<tr><th>'+d[1]+'</th><th>'+str(d[2][var])+'</th></tr>'
-        #t += '</tbody>'
-        return t
-
-
 @app.route('/application/<appkey>/device/<devid>/data/<var>/<dest>/<page>')
 def application_device_data(appkey, devid, var, dest, page):
+    dev = dd.get(appkey, devid)[1]
     if dest == 'graph':
         last = data.get_last_hours(appkey, devid, MAX_PG_ENTRIES_GRAPH_HOURS, int(page))
-        arr = '[["Time", "{}"],'.format(var)
+        arr = ''
         if last[0]:
-            for d in last[1]:
+            arr = '[["Time", "{}"],'.format(var)
+            last = [ddm.decode_datum(d, dev[3]) for d in last[1]]
+            for d in last:
                 arr += '[new Date('+str(d[0])+'*1000),'+str(d[2][var])+'],'
             arr += ']'
         return arr
@@ -485,7 +445,8 @@ def application_device_data(appkey, devid, var, dest, page):
         last = data.get_last_range(appkey, devid, [MAX_PG_ENTRIES_DATA, (int(page)-1)*MAX_PG_ENTRIES_DATA])
         t = ''
         if last[0]:
-            for d in last[1]:
+            last = [ddm.decode_datum(d, dev[3]) for d in last[1]]
+            for d in last:
                 t += '<tr><th>'+d[1]+'</th><th>'+str(d[2][var])+'</th></tr>'
         return t
 
@@ -629,8 +590,7 @@ def application_device_settings(appkey, devid):
 
             return render_template('new/public/device-settings.html', app=ap[1], dev=dev[1], models=ddm.MODELS)
         elif request.method == 'POST':
-            ddmin = misc.extract_ddm(request)
-            #res = dd.update(appkey, devid, request.form['devname'], request.form['devdesc'])
+            ddmin = ddm.extract(request)
             res = dd.update_ddm(appkey, devid, request.form['devname'], request.form['devdesc'], ddmin)
             
             if not res[0]: