From ac7582b6c392534406d16971d0a645514c21106a Mon Sep 17 00:00:00 2001
From: hichbra <hicham.brahimi@laposte.net>
Date: Wed, 3 Apr 2019 16:48:21 +0200
Subject: [PATCH] update fileToSql

---
 PC/FileToSql.py | 163 ++++++++++++++++++++++++++++++++++--------------
 PC/server.py    |   8 ++-
 2 files changed, 121 insertions(+), 50 deletions(-)

diff --git a/PC/FileToSql.py b/PC/FileToSql.py
index 449ca79..ed36e61 100644
--- a/PC/FileToSql.py
+++ b/PC/FileToSql.py
@@ -6,12 +6,38 @@ import csv
 import shutil
 from datetime import datetime
 
+def formatLigneCorrect(row):
+	try :
+		if len(row) < 7:
+			print("ligne incorrect -1")
+			return False
+
+		for i in range(1,7):
+			val = int(float(row[i]))
+
+		return True
+	except:
+		print("ligne incorrect -2")
+		return False
+
+# Un topic est correcte s'il est sous la forme : sourceFichier / positionColonne
+def formatTopicCorrect(topic):
+	try:
+		topic = topic.split("/")
+		source = str(topic[0])
+		col = int(topic[1])
+		return True
+	except:
+		return False
+
+
 log = open("logBDD.txt", "w")
 
+# ================ CONNECTION BDD ================== #
 try:
-	db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan",   
-		             user="firediag",         
-		             passwd="firediag$!",  
+	db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan",
+		             user="firediag",
+		             passwd="firediag$!",
 		             db="firediag");
 except Exception as ex:
 	log.write(str(datetime.now())+" ==> Exception Demarrage : "+str(type(ex))+" ("+str(ex.args)+")\n")
@@ -19,59 +45,102 @@ except Exception as ex:
 	log.close()
 	sys.exit(1)
 
-nomCapteur = "Moule A" # A0 / A1 / A2 / A3 / A4 / A5
+
+# ================ RECUPERATION TOPIC ================= #
+
+topics = []
+try:
+	#--- Recupere les topic des capteurs en base de donnees
+	requete = "SELECT DISTINCT topic FROM capteur"
+	cursor = db.cursor();
+	cursor.execute(requete)
+	for row in cursor.fetchall():
+		# On filtre les mauvais topics (qui ne peuvent pas etre split par /)
+		if formatTopicCorrect(row[0]):
+			topics.append(row[0])
+except Exception as ex:
+	log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n")
+	print(ex);
+	log.close()
+	sys.exit(1)
+
+print("Topics : ", topics)
+
+
+# ================ RECUPERATION ID ================= #
+
 tabId = {}
 
 try:
-	#--- Recupere les id des capteurs en base de donnees
-	for i in range(0,6):
-		requete = "SELECT DISTINCT id FROM capteur WHERE nom = '"+nomCapteur+""+str(i)+"'"
+	#--- Recupere les id des capteurs en fonction des topics
+	for topic in topics:
+		requete = "SELECT DISTINCT id FROM capteur WHERE topic = '"+topic+"'"
 		cursor = db.cursor();
 		cursor.execute(requete)
 		idTab = cursor.fetchall()
-		tabId["A"+str(i)] = int(idTab[0][0])
+		tabId[topic] = int(idTab[0][0])
 except Exception as ex:
 	log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n")
 	print(ex);
 	log.close()
 	sys.exit(1)
 
-print(tabId)
-
-FOLDER = "./data/"
-TRAITEES = "./traitees/"
-DIR = os.listdir(FOLDER)
-DIR.sort()
-
-cursor = db.cursor();
-
-#--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier 
-for f in DIR:
-	print(FOLDER+f)
-	with open(FOLDER+f,'r') as csvfile:			
-		try:
-			#--- Recupere la source du fichier		
-			id = csvfile.readline().strip().split(":")[1]
-			print(id)
-			
-			next(csvfile, None)
-			
-			valeurs = csv.reader(csvfile, delimiter=';')
-			for row in valeurs:
-				time = str(row[0]).strip()
-				
-				for i in range(0,6):
-					requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(time).strip()+"', '"+str(row[i+1]).strip()+"', '"+str(tabId["A"+str(i)])+"')"
-					cursor.execute(requete)
-				#print(row)
-		except Exception as ex:
-			db.rollback();
-			log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n")
-			print("rollback ");
-			print(ex);	
-	db.commit();
-	#os.remove(FOLDER+f)
-	shutil.move(FOLDER+f, TRAITEES+f)
-
-db.close();
-log.close()
+print("Id : ",tabId)
+
+# ================ File ==> SQL ================= #
+
+while(True):
+	FOLDER = "./data/"
+	TRAITEES = "./traitees/"
+	DIR = os.listdir(FOLDER)
+	DIR.sort()
+
+	cursor = db.cursor();
+
+	#--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier
+	for f in DIR:
+		print(FOLDER+f)
+		with open(FOLDER+f,'r') as csvfile:
+			try:
+				#--- Recupere la source du fichier
+				id = csvfile.readline().strip().split(":")[1]
+				print(id)
+
+				next(csvfile, None)
+
+				valeurs = csv.reader(csvfile, delimiter=';')
+				for row in valeurs:
+					if formatLigneCorrect(row):
+						date = str(row[0]).strip()
+						
+						for topic in tabId:
+							t = topic.split("/")
+							source = str(t[0])
+							col = int(t[1])
+							if id == source :					
+								requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(date).strip()+"', '"+str(row[col+1]).strip()+"', '"+str(tabId[topic])+"')"
+								cursor.execute(requete)
+
+			except KeyboardInterrupt:
+				db.rollback();
+				db.close();
+				log.close();
+				print("end")
+				sys.exit(0)
+
+			except Exception as ex:
+				db.rollback();
+				log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n")
+				print("rollback ");
+				print(ex);
+		db.commit();
+		#os.remove(FOLDER+f)
+		shutil.move(FOLDER+f, TRAITEES+f)
+	try:
+		print("============= Fin de cycle ============")
+		time.sleep(300) # Ajout des nouveaux fichiers en base toutes les 5 minutes
+	except KeyboardInterrupt:
+		db.close();
+		log.close();
+		print("end")
+		sys.exit(0)
diff --git a/PC/server.py b/PC/server.py
index 83a8844..89f00ad 100644
--- a/PC/server.py
+++ b/PC/server.py
@@ -3,6 +3,7 @@ import socket
 import sys
 import time
 from datetime import datetime
+import subprocess
 
 CHUNK_SIZE = 2048
 PORT = 20000
@@ -24,9 +25,9 @@ while True:
 
 		while True:
 			data = client.recv(CHUNK_SIZE)
-		
+
 			file.write(data)
-		
+
 			if len(data) == 0:
 				print("Fichier recu")
 				log.write(str(datetime.now())+" => Fichier Recu\n")
@@ -35,8 +36,9 @@ while True:
 				client.close()
 				server.close()
 				time.sleep(1) # attendre la liberation du port
+
 				break
-			print("  -> chunk")
+			#print("  -> chunk")
 
 	except KeyboardInterrupt:
 		if file is not None:
-- 
GitLab