periodic server maintanance task added. confirmed config messages and data...

periodic server maintanance task added. confirmed config messages and data files are removed during maintanance task

app/app/__init__.py

@@ -6,7 +6,6 @@ if app.config['ENV'] == 'production':
     app.config.from_object('config.ProductionConfig')
 else:
     app.config.from_object('config.DevelopmentConfig')
-    
 
 from app import views
-
+from app.helpers import maintainer

app/app/dao/pend/pend.py

@@ -38,3 +38,15 @@ def delete(cur, appkey, devid, msg):
     """
     cur.execute(query, (appkey, devid, msg))
     return (True,)
+
+@with_psql
+def delete_all_ack(cur):
+    query = """
+    DELETE FROM
+        pend_msgs
+    WHERE
+        ack = True
+    """
+    cur.execute(query, ())
+    return (True,)
+

app/app/helpers/maintainer.py

+from apscheduler.schedulers.background import BackgroundScheduler 
+from app import app, views
+import os
+
+def clean_data_folder():
+    try:
+        filelist = [f for f in os.listdir(app.config['DATA_DOWNLOAD_DIR_OS'])]
+        for f in filelist:
+            os.remove(app.config['DATA_DOWNLOAD_DIR_OS']+'/'+f)
+    except OSError:
+        pass
+
+def maintainer():
+    views.pend_delete_all_ack()
+    clean_data_folder()
+
+scheduler = BackgroundScheduler()
+job = scheduler.add_job(maintainer, 'interval', minutes=app.config['MAINTAINER_INTERVAL'])
+scheduler.start()

app/app/helpers/misc.py

@@ -66,3 +66,11 @@ def with_psql(f):
     
         return res
     return _with_psql
+
+def clean_data_folder():
+    try:
+        filelist = [f for f in os.listdir(app.config['DATA_DOWNLOAD_DIR_OS'])]
+        for f in filelist:
+            os.remove(app.config['DATA_DOWNLOAD_DIR_OS']+'/'+f)
+    except OSError:
+        pass

app/app/views.py

@@ -108,9 +108,9 @@ def app_():
             devs = dd.get_list(ap[1][1])
         
             try:
-                filelist = [f for f in os.listdir(app.config['DATA_DOWNLOAD_DIR']) if f.startswith(session['appkey'])]
+                filelist = [f for f in os.listdir(app.config['DATA_DOWNLOAD_DIR_OS'])]
                 for f in filelist:
-                    os.remove(app.config['DATA_DOWNLOAD_DIR']+'/'+f)
+                    os.remove(app.config['DATA_DOWNLOAD_DIR_OS']+'/'+f)
             except OSError:
                 pass
 
@@ -303,7 +303,7 @@ def data_csv():
 
         fn = session['appkey']+ '_' +str(session['devid'])+ '.csv'
 
-        with open('app/'+app.config['DATA_DOWNLOAD_DIR']+'/'+fn, 'w+') as f: 
+        with open(app.config['DATA_DOWNLOAD_DIR_OS']+'/'+fn, 'w+') as f: 
             f.write('utc,timestamp,')
             for d in dumpd[1][0][2]:
                 f.write(d)
@@ -320,3 +320,7 @@ def data_csv():
         return send_from_directory(app.config['DATA_DOWNLOAD_DIR'], fn, as_attachment=True)
     else:
         return redirect(utl_for('index'))
+
+
+def pend_delete_all_ack():
+    pend.delete_all_ack()

app/config.py

@@ -15,6 +15,10 @@ class Config(object):
 
     APPKEY_LENGTH = 8
     DATA_DOWNLOAD_DIR = 'data'
+    DATA_DOWNLOAD_DIR_OS = 'app/data'
+
+    # in minutes - 24 hours by default
+    MAINTAINER_INTERVAL = 1440
 
 class ProductionConfig(Config):
     pass

app/nohup.out

+[uWSGI] getting INI configuration from server.ini
+*** Starting uWSGI 2.0.18 (32bit) on [Tue Apr 21 12:24:28 2020] ***
+compiled with version: 6.3.0 20170516 on 20 April 2020 09:27:17
+os: Linux-4.19.66-v7+ #1253 SMP Thu Aug 15 11:49:46 BST 2019
+nodename: LoRaIk
+machine: armv7l
+clock source: unix
+detected number of CPU cores: 4
+current working directory: /home/pi/thso.server/app
+detected binary path: /home/pi/thso.server/env/bin/uwsgi
+!!! no internal routing support, rebuild with pcre support !!!
+your processes number limit is 6924
+your memory page size is 4096 bytes
+detected max file descriptor number: 1024
+lock engine: pthread robust mutexes
+thunder lock: disabled (you can enable it with --thunder-lock)
+uWSGI http bound on :8080 fd 4
+uwsgi socket 0 bound to TCP address 127.0.0.1:36961 (port auto-assigned) fd 3
+Python version: 3.5.3 (default, Sep 27 2018, 17:25:39)  [GCC 6.3.0 20170516]
+Python main interpreter initialized at 0x3a8038
+python threads support enabled
+your server socket listen backlog is limited to 100 connections
+your mercy for graceful operations on workers is 60 seconds
+mapped 261216 bytes (255 KB) for 8 cores
+*** Operational MODE: preforking+threaded ***
+WSGI app 0 (mountpoint='') ready in 1 seconds on interpreter 0x3a8038 pid: 11600 (default app)
+*** uWSGI is running in multiple interpreter mode ***
+spawned uWSGI master process (pid: 11600)
+spawned uWSGI worker 1 (pid: 11613, cores: 4)
+spawned uWSGI worker 2 (pid: 11614, cores: 4)
+spawned uWSGI http 1 (pid: 11618)
+SIGINT/SIGQUIT received...killing workers...
+gateway "uWSGI http 1" has been buried (pid: 11618)
+worker 1 buried after 1 seconds
+worker 2 buried after 1 seconds
+goodbye to uWSGI.
+[uWSGI] getting INI configuration from server.ini
+*** Starting uWSGI 2.0.18 (32bit) on [Tue Apr 21 12:24:40 2020] ***
+compiled with version: 6.3.0 20170516 on 20 April 2020 09:27:17
+os: Linux-4.19.66-v7+ #1253 SMP Thu Aug 15 11:49:46 BST 2019
+nodename: LoRaIk
+machine: armv7l
+clock source: unix
+detected number of CPU cores: 4
+current working directory: /home/pi/thso.server/app
+detected binary path: /home/pi/thso.server/env/bin/uwsgi
+!!! no internal routing support, rebuild with pcre support !!!
+your processes number limit is 6924
+your memory page size is 4096 bytes
+detected max file descriptor number: 1024
+lock engine: pthread robust mutexes
+thunder lock: disabled (you can enable it with --thunder-lock)
+uWSGI http bound on :8080 fd 4
+uwsgi socket 0 bound to TCP address 127.0.0.1:41041 (port auto-assigned) fd 3
+Python version: 3.5.3 (default, Sep 27 2018, 17:25:39)  [GCC 6.3.0 20170516]
+Python main interpreter initialized at 0x6ec038
+python threads support enabled
+your server socket listen backlog is limited to 100 connections
+your mercy for graceful operations on workers is 60 seconds
+mapped 261216 bytes (255 KB) for 8 cores
+*** Operational MODE: preforking+threaded ***
+WSGI app 0 (mountpoint='') ready in 1 seconds on interpreter 0x6ec038 pid: 11658 (default app)
+*** uWSGI is running in multiple interpreter mode ***
+spawned uWSGI master process (pid: 11658)
+spawned uWSGI worker 1 (pid: 11682, cores: 4)
+spawned uWSGI worker 2 (pid: 11683, cores: 4)
+spawned uWSGI http 1 (pid: 11688)
+[pid: 11683|app: 0|req: 1/1] 192.168.0.106 () {42 vars in 782 bytes} [Tue Apr 21 12:24:55 2020] GET / => generated 2110 bytes in 238 msecs (HTTP/1.1 200) 4 headers in 234 bytes (1 switches on core 0)
+[pid: 11682|app: 0|req: 1/2] 192.168.0.106 () {42 vars in 787 bytes} [Tue Apr 21 12:24:57 2020] GET /app?appkey=e6736c46 => generated 2814 bytes in 275 msecs (HTTP/1.1 200) 4 headers in 258 bytes (1 switches on core 0)
+[pid: 11682|app: 0|req: 2/3] 192.168.0.106 () {42 vars in 808 bytes} [Tue Apr 21 12:24:59 2020] GET /dev?id=1 => generated 2871 bytes in 187 msecs (HTTP/1.1 200) 4 headers in 258 bytes (1 switches on core 2)
+[pid: 11683|app: 0|req: 2/4] 192.168.0.106 () {42 vars in 798 bytes} [Tue Apr 21 12:25:03 2020] GET /dev-data => generated 13680 bytes in 200 msecs (HTTP/1.1 200) 3 headers in 96 bytes (1 switches on core 1)
+[pid: 11683|app: 0|req: 3/5] 192.168.0.106 () {42 vars in 780 bytes} [Tue Apr 21 12:25:26 2020] GET / => generated 2110 bytes in 116 msecs (HTTP/1.1 200) 4 headers in 234 bytes (1 switches on core 2)
+[pid: 11683|app: 0|req: 4/6] 192.168.0.106 () {40 vars in 708 bytes} [Tue Apr 21 16:10:59 2020] GET / => generated 2110 bytes in 101 msecs (HTTP/1.1 200) 3 headers in 95 bytes (1 switches on core 3)
+[pid: 11683|app: 0|req: 5/7] 192.168.0.106 () {42 vars in 787 bytes} [Tue Apr 21 16:11:02 2020] GET /app?appkey=e6736c46 => generated 2814 bytes in 193 msecs (HTTP/1.1 200) 4 headers in 258 bytes (1 switches on core 0)
+[pid: 11683|app: 0|req: 6/8] 192.168.0.106 () {42 vars in 808 bytes} [Tue Apr 21 16:11:04 2020] GET /dev?id=1 => generated 2870 bytes in 177 msecs (HTTP/1.1 200) 4 headers in 258 bytes (1 switches on core 1)
+[pid: 11683|app: 0|req: 7/9] 192.168.0.106 () {42 vars in 798 bytes} [Tue Apr 21 16:11:11 2020] GET /dev-data => generated 13674 bytes in 183 msecs (HTTP/1.1 200) 3 headers in 96 bytes (1 switches on core 2)
+[pid: 11682|app: 0|req: 3/10] 192.168.0.106 () {42 vars in 782 bytes} [Tue Apr 21 16:11:26 2020] GET / => generated 2110 bytes in 146 msecs (HTTP/1.1 200) 4 headers in 234 bytes (1 switches on core 1)
+gateway "uWSGI http 1" has been buried (pid: 11688)
+...brutally killing workers...
+worker 1 buried after 1 seconds
+worker 2 buried after 1 seconds
+binary reloading uWSGI...
+chdir() to /home/pi/thso.server/app
+closing all non-uwsgi socket fds > 2 (max_fd = 1024)...
+found fd 3 mapped to socket 0 (127.0.0.1:41041)
+running /home/pi/thso.server/env/bin/uwsgi
+[uWSGI] getting INI configuration from server.ini
+*** Starting uWSGI 2.0.18 (32bit) on [Wed Apr 22 12:12:44 2020] ***
+compiled with version: 6.3.0 20170516 on 20 April 2020 09:27:17
+os: Linux-4.19.66-v7+ #1253 SMP Thu Aug 15 11:49:46 BST 2019
+nodename: LoRaIk
+machine: armv7l
+clock source: unix
+detected number of CPU cores: 4
+current working directory: /home/pi/thso.server/app
+detected binary path: /home/pi/thso.server/env/bin/uwsgi
+!!! no internal routing support, rebuild with pcre support !!!
+your processes number limit is 6924
+your memory page size is 4096 bytes
+detected max file descriptor number: 1024
+lock engine: pthread robust mutexes
+thunder lock: disabled (you can enable it with --thunder-lock)
+uWSGI http bound on :8080 fd 5
+uwsgi socket 0 bound to TCP address 127.0.0.1:38741 (port auto-assigned) fd 4
+Python version: 3.5.3 (default, Sep 27 2018, 17:25:39)  [GCC 6.3.0 20170516]
+Python main interpreter initialized at 0x1de8f78
+python threads support enabled
+your server socket listen backlog is limited to 100 connections
+your mercy for graceful operations on workers is 60 seconds
+mapped 261216 bytes (255 KB) for 8 cores
+*** Operational MODE: preforking+threaded ***
+Traceback (most recent call last):
+  File "run.py", line 1, in <module>
+    from app import app        
+  File "./app/__init__.py", line 11, in <module>
+    from app import views
+  File "./app/views.py", line 6, in <module>
+    import app.dao.user.user as ud
+  File "./app/dao/user/user.py", line 1, in <module>
+    from app.helpers.misc import with_psql
+  File "./app/helpers/misc.py", line 5, in <module>
+    from app.dao.pend import delete_all_ack
+ImportError: cannot import name 'delete_all_ack'
+unable to load app 0 (mountpoint='') (callable not found or import error)
+*** no app loaded. going in full dynamic mode ***
+*** uWSGI is running in multiple interpreter mode ***
+gracefully (RE)spawned uWSGI master process (pid: 11658)
+spawned uWSGI worker 1 (pid: 13464, cores: 4)
+spawned uWSGI worker 2 (pid: 13465, cores: 4)
+spawned uWSGI http 1 (pid: 13469)
+gateway "uWSGI http 1" has been buried (pid: 13469)
+...brutally killing workers...
+worker 1 buried after 0 seconds
+worker 2 buried after 0 seconds
+binary reloading uWSGI...
+chdir() to /home/pi/thso.server/app
+closing all non-uwsgi socket fds > 2 (max_fd = 1024)...
+found fd 4 mapped to socket 0 (127.0.0.1:38741)
+running /home/pi/thso.server/env/bin/uwsgi
+[uWSGI] getting INI configuration from server.ini
+*** Starting uWSGI 2.0.18 (32bit) on [Wed Apr 22 12:12:58 2020] ***
+compiled with version: 6.3.0 20170516 on 20 April 2020 09:27:17
+os: Linux-4.19.66-v7+ #1253 SMP Thu Aug 15 11:49:46 BST 2019
+nodename: LoRaIk
+machine: armv7l
+clock source: unix
+detected number of CPU cores: 4
+current working directory: /home/pi/thso.server/app
+detected binary path: /home/pi/thso.server/env/bin/uwsgi
+!!! no internal routing support, rebuild with pcre support !!!
+your processes number limit is 6924
+your memory page size is 4096 bytes
+detected max file descriptor number: 1024
+lock engine: pthread robust mutexes
+thunder lock: disabled (you can enable it with --thunder-lock)
+uWSGI http bound on :8080 fd 5
+uwsgi socket 0 bound to TCP address 127.0.0.1:35833 (port auto-assigned) fd 3
+Python version: 3.5.3 (default, Sep 27 2018, 17:25:39)  [GCC 6.3.0 20170516]
+Python main interpreter initialized at 0x1598f78
+python threads support enabled
+your server socket listen backlog is limited to 100 connections
+your mercy for graceful operations on workers is 60 seconds
+mapped 261216 bytes (255 KB) for 8 cores
+*** Operational MODE: preforking+threaded ***
+Traceback (most recent call last):
+  File "run.py", line 1, in <module>
+    from app import app        
+  File "./app/__init__.py", line 11, in <module>
+    from app import views
+  File "./app/views.py", line 6, in <module>
+    import app.dao.user.user as ud
+  File "./app/dao/user/user.py", line 1, in <module>
+    from app.helpers.misc import with_psql
+  File "./app/helpers/misc.py", line 5, in <module>
+    from app.dao.pend import delete_all_ack
+ImportError: cannot import name 'delete_all_ack'
+unable to load app 0 (mountpoint='') (callable not found or import error)
+*** no app loaded. going in full dynamic mode ***
+*** uWSGI is running in multiple interpreter mode ***
+gracefully (RE)spawned uWSGI master process (pid: 11658)
+spawned uWSGI worker 1 (pid: 13545, cores: 4)
+spawned uWSGI worker 2 (pid: 13546, cores: 4)
+spawned uWSGI http 1 (pid: 13550)
+gateway "uWSGI http 1" has been buried (pid: 13550)
+...brutally killing workers...
+worker 1 buried after 1 seconds
+worker 2 buried after 1 seconds
+binary reloading uWSGI...
+chdir() to /home/pi/thso.server/app
+closing all non-uwsgi socket fds > 2 (max_fd = 1024)...
+found fd 3 mapped to socket 0 (127.0.0.1:35833)
+running /home/pi/thso.server/env/bin/uwsgi
+[uWSGI] getting INI configuration from server.ini
+*** Starting uWSGI 2.0.18 (32bit) on [Wed Apr 22 12:13:01 2020] ***
+compiled with version: 6.3.0 20170516 on 20 April 2020 09:27:17
+os: Linux-4.19.66-v7+ #1253 SMP Thu Aug 15 11:49:46 BST 2019
+nodename: LoRaIk
+machine: armv7l
+clock source: unix
+detected number of CPU cores: 4
+current working directory: /home/pi/thso.server/app
+detected binary path: /home/pi/thso.server/env/bin/uwsgi
+!!! no internal routing support, rebuild with pcre support !!!
+your processes number limit is 6924
+your memory page size is 4096 bytes
+detected max file descriptor number: 1024
+lock engine: pthread robust mutexes
+thunder lock: disabled (you can enable it with --thunder-lock)
+uWSGI http bound on :8080 fd 5
+uwsgi socket 0 bound to TCP address 127.0.0.1:34889 (port auto-assigned) fd 4
+Python version: 3.5.3 (default, Sep 27 2018, 17:25:39)  [GCC 6.3.0 20170516]
+Python main interpreter initialized at 0x150bf78
+python threads support enabled
+your server socket listen backlog is limited to 100 connections
+your mercy for graceful operations on workers is 60 seconds
+mapped 261216 bytes (255 KB) for 8 cores
+*** Operational MODE: preforking+threaded ***
+Traceback (most recent call last):
+  File "run.py", line 1, in <module>
+    from app import app        
+  File "./app/__init__.py", line 11, in <module>
+    from app import views
+  File "./app/views.py", line 6, in <module>
+    import app.dao.user.user as ud
+  File "./app/dao/user/user.py", line 1, in <module>
+    from app.helpers.misc import with_psql
+  File "./app/helpers/misc.py", line 5, in <module>
+    from app.dao.pend import delete_all_ack
+ImportError: cannot import name 'delete_all_ack'
+unable to load app 0 (mountpoint='') (callable not found or import error)
+*** no app loaded. going in full dynamic mode ***
+*** uWSGI is running in multiple interpreter mode ***
+gracefully (RE)spawned uWSGI master process (pid: 11658)
+spawned uWSGI worker 1 (pid: 13558, cores: 4)
+spawned uWSGI worker 2 (pid: 13559, cores: 4)
+spawned uWSGI http 1 (pid: 13563)
+Wed Apr 22 12:13:46 2020 - uWSGI worker 1 screams: UAAAAAAH my master disconnected: i will kill myself !!!
+Wed Apr 22 12:13:46 2020 - uWSGI worker 1 screams: UAAAAAAH my master disconnected: i will kill myself !!!
+Wed Apr 22 12:13:46 2020 - uWSGI worker 2 screams: UAAAAAAH my master disconnected: i will kill myself !!!
+Wed Apr 22 12:13:46 2020 - uWSGI worker 2 screams: UAAAAAAH my master disconnected: i will kill myself !!!

app/requirements.txt

+APScheduler==3.6.3
 bcrypt==3.1.7
 cffi==1.14.0
 click==7.1.1
@@ -7,6 +8,8 @@ Jinja2==2.11.2
 MarkupSafe==1.1.1
 psycopg2==2.8.5
 pycparser==2.20
+pytz==2019.3
 six==1.14.0
+tzlocal==2.0.0
 uWSGI==2.0.18
 Werkzeug==1.0.1

env/lib/python3.5/site-packages/APScheduler-3.6.3.dist-info/INSTALLER

+pip

env/lib/python3.5/site-packages/APScheduler-3.6.3.dist-info/LICENSE.txt

+This is the MIT license: http://www.opensource.org/licenses/mit-license.php
+
+Copyright (c) Alex Grönholm
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or
+substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.