startSimus.py 2.35 KB
Newer Older
1 2 3 4
from subprocess import run, Popen, PIPE
import multiprocessing
import time
from datetime import datetime
5
from datetime import timedelta
6 7 8 9 10

def nbContainers():
	result=run("docker ps -aq | wc -l", shell=True, stdout=PIPE).stdout.decode('utf-8')
	return int(result)

11
def backup(strDate):
12 13 14
	running_containers = run("docker ps --format '{{.Names}}'", shell=True, stdout=PIPE, stderr=PIPE).stdout.decode('utf-8').split("\n")
	for c in running_containers:
		if c: # if c is not empty (it happens for the last element in p)
15
			run("docker checkpoint create  --leave-running=true --checkpoint-dir=/mnt/TDEMARE/"+strDate+"/"+str(c)+"/Backup/ " + str(c) + " " + "checkpoint_" + str(c), shell=True)
16

17
def checkForBackupNeedsAndWait(strDate):
18 19
	# Should we backup the simulations ?
	timeBetweenTwoBackup = timedelta(minutes=2) # We have one day between two backups
20 21
	now = datetime.now()
	if now > (lastBackup + timeBetweenTwoBackup):
22
		backup(strDate)
23

24
	# Sleep a bit: don't need to be awaken all the time
25 26
	time.sleep(350)

27
# Starting date
ThibautDemare's avatar
ThibautDemare committed
28
startingDate = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
29

30
# The number of available CPU
31 32
nbCpus = multiprocessing.cpu_count()

33
# The date of the last backup done
34 35
lastBackup = datetime.now()

36 37
# The index of the first simulation
startingIndex = 1
38

39 40 41 42 43 44 45
# The index of the last simulation
endingIndex = 4

# The index of the simulation to start next
nbSimStarted = startingIndex

while nbSimStarted <= endingIndex:
46 47 48 49 50 51 52 53 54 55 56

	resultPath = "run"+str(nbSimStarted)
	run("cp -r ./source ./"+str(resultPath), shell = True)

	command = ["docker run --rm --name run"+str(nbSimStarted)+" -v $(pwd)/"+str(resultPath)+"/simulation_configs:/simulation_configs -v $(pwd)/"+str(resultPath)+"/Backup:/Backup -v $(pwd)/"+str(resultPath)+"/Agents:/Agents -v $(pwd)/"+str(resultPath)+"/output:/output -v $(pwd)/"+str(resultPath)+"/CSV:/CSV -v $(pwd)/"+str(resultPath)+"/bd:/bd -v $(pwd)/"+str(resultPath)+"/DALSim:/DALSim gama > out"+str(nbSimStarted)+" 2>&1"]
	Popen(command, shell = True)
	nbSimStarted += 1
	time.sleep(30) # wait 30 sec to make sure the container is really started

	# Loop if the current number of running containers is greater or equal than the number of available cpus
	while nbCpus <= nbContainers():
57
		checkForBackupNeedsAndWait(startingDate)
58 59 60 61


while nbContainers() > 0:
	# We started all the containers, but we still have to wait until their end
62
	checkForBackupNeedsAndWait(startingDate)