From ac7582b6c392534406d16971d0a645514c21106a Mon Sep 17 00:00:00 2001 From: hichbra <hicham.brahimi@laposte.net> Date: Wed, 3 Apr 2019 16:48:21 +0200 Subject: [PATCH] update fileToSql --- PC/FileToSql.py | 163 ++++++++++++++++++++++++++++++++++-------------- PC/server.py | 8 ++- 2 files changed, 121 insertions(+), 50 deletions(-) diff --git a/PC/FileToSql.py b/PC/FileToSql.py index 449ca79..ed36e61 100644 --- a/PC/FileToSql.py +++ b/PC/FileToSql.py @@ -6,12 +6,38 @@ import csv import shutil from datetime import datetime +def formatLigneCorrect(row): + try : + if len(row) < 7: + print("ligne incorrect -1") + return False + + for i in range(1,7): + val = int(float(row[i])) + + return True + except: + print("ligne incorrect -2") + return False + +# Un topic est correcte s'il est sous la forme : sourceFichier / positionColonne +def formatTopicCorrect(topic): + try: + topic = topic.split("/") + source = str(topic[0]) + col = int(topic[1]) + return True + except: + return False + + log = open("logBDD.txt", "w") +# ================ CONNECTION BDD ================== # try: - db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan", - user="firediag", - passwd="firediag$!", + db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan", + user="firediag", + passwd="firediag$!", db="firediag"); except Exception as ex: log.write(str(datetime.now())+" ==> Exception Demarrage : "+str(type(ex))+" ("+str(ex.args)+")\n") @@ -19,59 +45,102 @@ except Exception as ex: log.close() sys.exit(1) -nomCapteur = "Moule A" # A0 / A1 / A2 / A3 / A4 / A5 + +# ================ RECUPERATION TOPIC ================= # + +topics = [] +try: + #--- Recupere les topic des capteurs en base de donnees + requete = "SELECT DISTINCT topic FROM capteur" + cursor = db.cursor(); + cursor.execute(requete) + for row in cursor.fetchall(): + # On filtre les mauvais topics (qui ne peuvent pas etre split par /) + if formatTopicCorrect(row[0]): + topics.append(row[0]) +except Exception as ex: + log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n") + print(ex); + log.close() + sys.exit(1) + +print("Topics : ", topics) + + +# ================ RECUPERATION ID ================= # + tabId = {} try: - #--- Recupere les id des capteurs en base de donnees - for i in range(0,6): - requete = "SELECT DISTINCT id FROM capteur WHERE nom = '"+nomCapteur+""+str(i)+"'" + #--- Recupere les id des capteurs en fonction des topics + for topic in topics: + requete = "SELECT DISTINCT id FROM capteur WHERE topic = '"+topic+"'" cursor = db.cursor(); cursor.execute(requete) idTab = cursor.fetchall() - tabId["A"+str(i)] = int(idTab[0][0]) + tabId[topic] = int(idTab[0][0]) except Exception as ex: log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n") print(ex); log.close() sys.exit(1) -print(tabId) - -FOLDER = "./data/" -TRAITEES = "./traitees/" -DIR = os.listdir(FOLDER) -DIR.sort() - -cursor = db.cursor(); - -#--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier -for f in DIR: - print(FOLDER+f) - with open(FOLDER+f,'r') as csvfile: - try: - #--- Recupere la source du fichier - id = csvfile.readline().strip().split(":")[1] - print(id) - - next(csvfile, None) - - valeurs = csv.reader(csvfile, delimiter=';') - for row in valeurs: - time = str(row[0]).strip() - - for i in range(0,6): - requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(time).strip()+"', '"+str(row[i+1]).strip()+"', '"+str(tabId["A"+str(i)])+"')" - cursor.execute(requete) - #print(row) - except Exception as ex: - db.rollback(); - log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n") - print("rollback "); - print(ex); - db.commit(); - #os.remove(FOLDER+f) - shutil.move(FOLDER+f, TRAITEES+f) - -db.close(); -log.close() +print("Id : ",tabId) + +# ================ File ==> SQL ================= # + +while(True): + FOLDER = "./data/" + TRAITEES = "./traitees/" + DIR = os.listdir(FOLDER) + DIR.sort() + + cursor = db.cursor(); + + #--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier + for f in DIR: + print(FOLDER+f) + with open(FOLDER+f,'r') as csvfile: + try: + #--- Recupere la source du fichier + id = csvfile.readline().strip().split(":")[1] + print(id) + + next(csvfile, None) + + valeurs = csv.reader(csvfile, delimiter=';') + for row in valeurs: + if formatLigneCorrect(row): + date = str(row[0]).strip() + + for topic in tabId: + t = topic.split("/") + source = str(t[0]) + col = int(t[1]) + if id == source : + requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(date).strip()+"', '"+str(row[col+1]).strip()+"', '"+str(tabId[topic])+"')" + cursor.execute(requete) + + except KeyboardInterrupt: + db.rollback(); + db.close(); + log.close(); + print("end") + sys.exit(0) + + except Exception as ex: + db.rollback(); + log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n") + print("rollback "); + print(ex); + db.commit(); + #os.remove(FOLDER+f) + shutil.move(FOLDER+f, TRAITEES+f) + try: + print("============= Fin de cycle ============") + time.sleep(300) # Ajout des nouveaux fichiers en base toutes les 5 minutes + except KeyboardInterrupt: + db.close(); + log.close(); + print("end") + sys.exit(0) diff --git a/PC/server.py b/PC/server.py index 83a8844..89f00ad 100644 --- a/PC/server.py +++ b/PC/server.py @@ -3,6 +3,7 @@ import socket import sys import time from datetime import datetime +import subprocess CHUNK_SIZE = 2048 PORT = 20000 @@ -24,9 +25,9 @@ while True: while True: data = client.recv(CHUNK_SIZE) - + file.write(data) - + if len(data) == 0: print("Fichier recu") log.write(str(datetime.now())+" => Fichier Recu\n") @@ -35,8 +36,9 @@ while True: client.close() server.close() time.sleep(1) # attendre la liberation du port + break - print(" -> chunk") + #print(" -> chunk") except KeyboardInterrupt: if file is not None: -- GitLab