Commit 1923908a authored by Martin Schorb's avatar Martin Schorb
Browse files

sparkslurm status fix

parent 27ac42e0
......@@ -80,7 +80,7 @@ def update_status(n,click,run_state,logfile,module,thispage):
(r_status['status'], link) = launch_jobs.status(run_state)
if not link == '':
status_href = link
status_href = link.split('__')[-1]
status_style = {}
......@@ -211,13 +211,13 @@ def run_state(launch_trigger,status_trigger,launch_in,status_in):
# print(trigger)
if 'launch' in trigger:
# print('launch triggered state:')
# print(launch_in)
print('launch triggered state:')
print(launch_in)
out = launch_in
else:
# print('status triggered state:')
# print(status_in)
print('status triggered state:')
print(status_in)
out = status_in.copy()
......
......@@ -92,6 +92,12 @@ n_cpu_standalone = 8
n_cpu_spark = 200
cpu_pernode_spark = 15
spark_port = '8080'
spark_job_port = '4040'
# runtime parameters
mipmaps=dict()
mipmaps['min/Gpix/CPU'] = 6
......
......@@ -38,7 +38,6 @@ def args2string(args,separator='='):
def status(run_state):
run_state.update({'id':31801858,'type':'sparkslurm'})
res_status,link = checkstatus(run_state)
# print(run_state)
# print('res_status:')
......@@ -109,11 +108,10 @@ def checkstatus(run_state):
def cluster_status(run_state):
my_env = os.environ.copy()
out_stat=list()
sp_master=''
link=''
j_id = run_state['id']
# print('JOB-ID:')
# print(j_id)
print(run_state)
if j_id=='':
return 'wait'
......@@ -185,14 +183,15 @@ def cluster_status(run_state):
masterhost = jobstat[2]
slurm_stat = jobstat[1]
print(command)
if 'RUNNING' in slurm_stat:
sp_masterfile = os.path.join(logfile.rsplit(os.extsep)[0],'spark-master-' + str(j_id),'master')
with open(sp_masterfile) as f: sp_master=f.read().strip('\n')
link = '__' + sp_master
url = sp_master + '/json/'
link = '__http://' + sp_master + ':' + params.spark_job_port
url = 'http://' + sp_master + ':' + params.spark_port + '/json/'
try:
sp_query = requests.get(url).json()
......@@ -241,7 +240,7 @@ def cluster_status(run_state):
elif 'CANCELLED' in slurm_stat:
out_stat.append('cancelled')
return out_stat[0],sp_master
return out_stat[0],link
def canceljobs(run_state):
......@@ -403,7 +402,7 @@ def run(target='standalone',
f.write(jobid)
jobid=jobid.strip('\n')[jobid.rfind(' ')+1:]
print('---'+jobid)
return jobid
......
......@@ -87,7 +87,7 @@ mkdir $LOGDIR/$JOB
# SET UP ENV for the spark run
echo $MASTER_WEB > $LOGDIR/$JOB/master
echo $MASTER_IP > $LOGDIR/$JOB/master
export SPARK_LOG_DIR="$LOGDIR/$JOB/logs"
export SPARK_WORKER_DIR="$LOGDIR/$JOB/worker"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment