Commit ac7582b6 authored by hichbra's avatar hichbra

update fileToSql

parent a95a853f
......@@ -6,12 +6,38 @@ import csv
import shutil
from datetime import datetime
def formatLigneCorrect(row):
try :
if len(row) < 7:
print("ligne incorrect -1")
return False
for i in range(1,7):
val = int(float(row[i]))
return True
except:
print("ligne incorrect -2")
return False
# Un topic est correcte s'il est sous la forme : sourceFichier / positionColonne
def formatTopicCorrect(topic):
try:
topic = topic.split("/")
source = str(topic[0])
col = int(topic[1])
return True
except:
return False
log = open("logBDD.txt", "w")
# ================ CONNECTION BDD ================== #
try:
db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan",
user="firediag",
passwd="firediag$!",
db = MySQLdb.connect(host="localhost",#"pil-09.univlehavre.lan",
user="firediag",
passwd="firediag$!",
db="firediag");
except Exception as ex:
log.write(str(datetime.now())+" ==> Exception Demarrage : "+str(type(ex))+" ("+str(ex.args)+")\n")
......@@ -19,59 +45,102 @@ except Exception as ex:
log.close()
sys.exit(1)
nomCapteur = "Moule A" # A0 / A1 / A2 / A3 / A4 / A5
# ================ RECUPERATION TOPIC ================= #
topics = []
try:
#--- Recupere les topic des capteurs en base de donnees
requete = "SELECT DISTINCT topic FROM capteur"
cursor = db.cursor();
cursor.execute(requete)
for row in cursor.fetchall():
# On filtre les mauvais topics (qui ne peuvent pas etre split par /)
if formatTopicCorrect(row[0]):
topics.append(row[0])
except Exception as ex:
log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n")
print(ex);
log.close()
sys.exit(1)
print("Topics : ", topics)
# ================ RECUPERATION ID ================= #
tabId = {}
try:
#--- Recupere les id des capteurs en base de donnees
for i in range(0,6):
requete = "SELECT DISTINCT id FROM capteur WHERE nom = '"+nomCapteur+""+str(i)+"'"
#--- Recupere les id des capteurs en fonction des topics
for topic in topics:
requete = "SELECT DISTINCT id FROM capteur WHERE topic = '"+topic+"'"
cursor = db.cursor();
cursor.execute(requete)
idTab = cursor.fetchall()
tabId["A"+str(i)] = int(idTab[0][0])
tabId[topic] = int(idTab[0][0])
except Exception as ex:
log.write(str(datetime.now())+" ==> Exception Init : "+str(type(ex))+" ("+str(ex.args)+")\n")
print(ex);
log.close()
sys.exit(1)
print(tabId)
FOLDER = "./data/"
TRAITEES = "./traitees/"
DIR = os.listdir(FOLDER)
DIR.sort()
cursor = db.cursor();
#--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier
for f in DIR:
print(FOLDER+f)
with open(FOLDER+f,'r') as csvfile:
try:
#--- Recupere la source du fichier
id = csvfile.readline().strip().split(":")[1]
print(id)
next(csvfile, None)
valeurs = csv.reader(csvfile, delimiter=';')
for row in valeurs:
time = str(row[0]).strip()
for i in range(0,6):
requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(time).strip()+"', '"+str(row[i+1]).strip()+"', '"+str(tabId["A"+str(i)])+"')"
cursor.execute(requete)
#print(row)
except Exception as ex:
db.rollback();
log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n")
print("rollback ");
print(ex);
db.commit();
#os.remove(FOLDER+f)
shutil.move(FOLDER+f, TRAITEES+f)
db.close();
log.close()
print("Id : ",tabId)
# ================ File ==> SQL ================= #
while(True):
FOLDER = "./data/"
TRAITEES = "./traitees/"
DIR = os.listdir(FOLDER)
DIR.sort()
cursor = db.cursor();
#--- Parcours les fichiers, ajoute les valeurs en base et supprime le fichier
for f in DIR:
print(FOLDER+f)
with open(FOLDER+f,'r') as csvfile:
try:
#--- Recupere la source du fichier
id = csvfile.readline().strip().split(":")[1]
print(id)
next(csvfile, None)
valeurs = csv.reader(csvfile, delimiter=';')
for row in valeurs:
if formatLigneCorrect(row):
date = str(row[0]).strip()
for topic in tabId:
t = topic.split("/")
source = str(t[0])
col = int(t[1])
if id == source :
requete = "INSERT INTO `valeur` (`temps`, `valeur`, `idCapteur`) VALUES ('"+str(date).strip()+"', '"+str(row[col+1]).strip()+"', '"+str(tabId[topic])+"')"
cursor.execute(requete)
except KeyboardInterrupt:
db.rollback();
db.close();
log.close();
print("end")
sys.exit(0)
except Exception as ex:
db.rollback();
log.write(str(datetime.now())+" ==> Exception Requete : "+str(type(ex))+" ("+str(ex.args)+")\n")
print("rollback ");
print(ex);
db.commit();
#os.remove(FOLDER+f)
shutil.move(FOLDER+f, TRAITEES+f)
try:
print("============= Fin de cycle ============")
time.sleep(300) # Ajout des nouveaux fichiers en base toutes les 5 minutes
except KeyboardInterrupt:
db.close();
log.close();
print("end")
sys.exit(0)
......@@ -3,6 +3,7 @@ import socket
import sys
import time
from datetime import datetime
import subprocess
CHUNK_SIZE = 2048
PORT = 20000
......@@ -24,9 +25,9 @@ while True:
while True:
data = client.recv(CHUNK_SIZE)
file.write(data)
if len(data) == 0:
print("Fichier recu")
log.write(str(datetime.now())+" => Fichier Recu\n")
......@@ -35,8 +36,9 @@ while True:
client.close()
server.close()
time.sleep(1) # attendre la liberation du port
break
print(" -> chunk")
#print(" -> chunk")
except KeyboardInterrupt:
if file is not None:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment