From ac7582b6c392534406d16971d0a645514c21106a Mon Sep 17 00:00:00 2001
From: hichbra <hicham.brahimi@laposte.net>
Date: Wed, 3 Apr 2019 16:48:21 +0200
Subject: [PATCH] update fileToSql
---
PC/FileToSql.py | 163 ++++++++++++++++++++++++++++++++++--------------
PC/server.py | 8 ++-
2 files changed, 121 insertions(+), 50 deletions(-)
diff --git a/PC/FileToSql.py b/PC/FileToSql.py
index 449ca79..ed36e61 100644
--- a/PC/FileToSql.py
+++ b/PC/FileToSql.py
@@ -6,12 +6,38 @@ import csv
import shutil
from datetime import datetime
+def formatLigneCorrect(row):
+ try :
+ if len(row) < 7:
+ print("ligne incorrect -1")
+ return False
+
+ for i in range(1,7):
+ val = int(float(row[i]))
+
+ return True
+ except:
+ print("ligne incorrect -2")
+ return False
+
+# Un topic est correcte s'il est sous la forme : sourceFichier / positionColonne
+def formatTopicCorrect(topic):
+ try:
+ topic = topic.split("/")
+ source = str(topic[0])
+ col = int(topic[1])
+ return True
+ except:
+ return False
+
+
log = open("logBDD.txt", "w")
+# ================ CONNECTION BDD ================== #
try:
- db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan",
- user="firediag",
- passwd="firediag$!",
+ db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan",
+ user="firediag",
+ passwd="firediag$!",
db="firediag");
except Exception as ex:
log.write(str(datetime.now())+" ==> Exception Demarrage : "+str(type(ex))+" ("+str(ex.args)+")\n")
@@ -19,59 +45,102 @@ except Exception as ex:
log.close()
sys.exit(1)
-nomCapteur = "Moule A" # A0 / A1 / A2 / A3 / A4 / A5
+
+# ================ RECUPERATION TOPIC ================= #
+
+topics = []
+try:
+ #--- Recupere les topic des capteurs en base de donnees
+ requete = "SELECT DISTINCT topic FROM capteur"
+ cursor = db.cursor();
+ cursor.execute(requete)
+ for row in cursor.fetchall():
+ # On filtre les mauvais topics (qui ne peuvent pas etre split par /)
+ if formatTopicCorrect(row[0]):
+ topics.append(row[0])
+except Exception as ex:
+ log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n")
+ print(ex);
+ log.close()
+ sys.exit(1)
+
+print("Topics : ", topics)
+
+
+# ================ RECUPERATION ID ================= #
+
tabId = {}
try:
- #--- Recupere les id des capteurs en base de donnees
- for i in range(0,6):
- requete = "SELECT DISTINCT id FROM capteur WHERE nom = '"+nomCapteur+""+str(i)+"'"
+ #--- Recupere les id des capteurs en fonction des topics
+ for topic in topics:
+ requete = "SELECT DISTINCT id FROM capteur WHERE topic = '"+topic+"'"
cursor = db.cursor();
cursor.execute(requete)
idTab = cursor.fetchall()
- tabId["A"+str(i)] = int(idTab[0][0])
+ tabId[topic] = int(idTab[0][0])
except Exception as ex:
log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n")
print(ex);
log.close()
sys.exit(1)
-print(tabId)
-
-FOLDER = "./data/"
-TRAITEES = "./traitees/"
-DIR = os.listdir(FOLDER)
-DIR.sort()
-
-cursor = db.cursor();
-
-#--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier
-for f in DIR:
- print(FOLDER+f)
- with open(FOLDER+f,'r') as csvfile:
- try:
- #--- Recupere la source du fichier
- id = csvfile.readline().strip().split(":")[1]
- print(id)
-
- next(csvfile, None)
-
- valeurs = csv.reader(csvfile, delimiter=';')
- for row in valeurs:
- time = str(row[0]).strip()
-
- for i in range(0,6):
- requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(time).strip()+"', '"+str(row[i+1]).strip()+"', '"+str(tabId["A"+str(i)])+"')"
- cursor.execute(requete)
- #print(row)
- except Exception as ex:
- db.rollback();
- log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n")
- print("rollback ");
- print(ex);
- db.commit();
- #os.remove(FOLDER+f)
- shutil.move(FOLDER+f, TRAITEES+f)
-
-db.close();
-log.close()
+print("Id : ",tabId)
+
+# ================ File ==> SQL ================= #
+
+while(True):
+ FOLDER = "./data/"
+ TRAITEES = "./traitees/"
+ DIR = os.listdir(FOLDER)
+ DIR.sort()
+
+ cursor = db.cursor();
+
+ #--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier
+ for f in DIR:
+ print(FOLDER+f)
+ with open(FOLDER+f,'r') as csvfile:
+ try:
+ #--- Recupere la source du fichier
+ id = csvfile.readline().strip().split(":")[1]
+ print(id)
+
+ next(csvfile, None)
+
+ valeurs = csv.reader(csvfile, delimiter=';')
+ for row in valeurs:
+ if formatLigneCorrect(row):
+ date = str(row[0]).strip()
+
+ for topic in tabId:
+ t = topic.split("/")
+ source = str(t[0])
+ col = int(t[1])
+ if id == source :
+ requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(date).strip()+"', '"+str(row[col+1]).strip()+"', '"+str(tabId[topic])+"')"
+ cursor.execute(requete)
+
+ except KeyboardInterrupt:
+ db.rollback();
+ db.close();
+ log.close();
+ print("end")
+ sys.exit(0)
+
+ except Exception as ex:
+ db.rollback();
+ log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n")
+ print("rollback ");
+ print(ex);
+ db.commit();
+ #os.remove(FOLDER+f)
+ shutil.move(FOLDER+f, TRAITEES+f)
+ try:
+ print("============= Fin de cycle ============")
+ time.sleep(300) # Ajout des nouveaux fichiers en base toutes les 5 minutes
+ except KeyboardInterrupt:
+ db.close();
+ log.close();
+ print("end")
+ sys.exit(0)
diff --git a/PC/server.py b/PC/server.py
index 83a8844..89f00ad 100644
--- a/PC/server.py
+++ b/PC/server.py
@@ -3,6 +3,7 @@ import socket
import sys
import time
from datetime import datetime
+import subprocess
CHUNK_SIZE = 2048
PORT = 20000
@@ -24,9 +25,9 @@ while True:
while True:
data = client.recv(CHUNK_SIZE)
-
+
file.write(data)
-
+
if len(data) == 0:
print("Fichier recu")
log.write(str(datetime.now())+" => Fichier Recu\n")
@@ -35,8 +36,9 @@ while True:
client.close()
server.close()
time.sleep(1) # attendre la liberation du port
+
break
- print(" -> chunk")
+ #print(" -> chunk")
except KeyboardInterrupt:
if file is not None:
--
GitLab