Hi Shankar, <div><br></div><div>Can you run Britton's suggestion through qsub?</div><div><br></div><div>Sam<br><br><div class="gmail_quote">On Thu, Apr 14, 2011 at 2:00 PM, Agarwal, Shankar <span dir="ltr"><<a href="mailto:sagarwal@ku.edu" target="_blank">sagarwal@ku.edu</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Hi all,<br>
<br>
The volume_render.py is reproduced at the end of this email. As Stephen noted, mpi4py is called internally in the script. I am successfully able to run volume_render.py as...<br>
<br>
mpirun -np 2 python volume_render.py<br>
<br>
(Britton's suggestion works fine. I see 0 and 1 printed out for the 2 cpus).<br>
<br>
<br>
Then I commented out the mpi related lines and was successfully able to run volume_render.py serially as well...<br>
<br>
python volume_render.py<br>
<br>
<br>
All the above was done on the login node. Then I repeated the above through qsub. First, I started an interactive session...<br>
<br>
qsub -I -V -l walltime=00:30:00,ncpus=6,mem=20gb -q debug<br>
<br>
Then I tried the serial version of volume_render.py and was successful...<br>
[sagarwal@ember-cmp1 ~/junk]$ python volume_render.py<br>
<br>
<br>
But the mpi4py version failed as below...<br>
<br>
[sagarwal@ember-cmp1 ~/junk]$ mpirun -np 2 python volume_render.py<br>
<div>MPI: MPI_COMM_WORLD rank 1 has terminated without calling MPI_Finalize()<br>
MPI: aborting job<br>
MPI: Received signal 11<br>
<br>
<br>
</div>So basically, mpi4py works fine only when I am on the login node. Me confused!<br>
<br>
Shankar<br>
<br>
<br>
<br>
-----------------------------------------------------------------------------------------------------------------------------------------<br>
<div>import matplotlib;matplotlib.use("Agg");import pylab<br>
import numpy as na<br>
import time<br>
from yt.config import ytcfg; ytcfg["yt","loglevel"] = '50' ; ytcfg["lagos","serialize"] = "False"<br>
<br>
from yt.mods import *<br>
from yt.extensions.volume_rendering import *<br>
from yt.funcs import *<br>
mh = na.log10(1.67e-24)<br>
from mpi4py import MPI<br>
</div><div>print MPI.COMM_WORLD.rank<br>
<br>
</div>def use_cluster(n = 1, frame=0, rotframes = 50):<br>
pf = EnzoStaticOutput("RD%04i/RedshiftOutput%04i" % (n,n) )<br>
<br>
z = pf.get_parameter('CosmologyCurrentRedshift')<br>
L = [na.sin(2*na.pi*frame/rotframes),na.cos(2*na.pi*frame/rotframes),0.4]<br>
<br>
c = [0.5,0.5,0.5]<br>
W = 0.8<br>
<br>
ncolors = 8<br>
max = -27.0+ na.log10((1.+z)**3)<br>
min = -32.0+ na.log10((1.+z)**3)<br>
tf = ColorTransferFunction((min-3., max+3.))<br>
<br>
valrange = na.linspace(min, max, ncolors)<br>
hues = range(0,256,256/ncolors)<br>
alphas = na.logspace(-2.0, 0.0, ncolors)<br>
width = 0.01*(max-min)/ncolors<br>
<br>
for i,val in enumerate(valrange):<br>
# thisrgb = yt.raven.ColorMaps.raven_colormaps['kamae'](hues[i])<br>
thisrgb = pylab.cm.spectral(hues[i])<br>
tf.add_gaussian(val, width, [thisrgb[0], thisrgb[1],<br>
thisrgb[2], alphas[i]])<br>
<br>
return (pf, L, c, W, tf)<br>
<br>
myrank = MPI.COMM_WORLD.Get_rank()<br>
procs = MPI.COMM_WORLD.Get_size()<br>
rocnm = MPI.Get_processor_name()<br>
<br>
Nvec = 1024<br>
for i in range(0,3):<br>
if (i % nprocs != myrank):<br>
continue<br>
pre_name = 'rot_evolve_%04i' % i<br>
if os.path.isfile("%s_alt_rgb.png" % pre_name): continue<br>
print 'On data dump %04i' % i<br>
pf, L, c, W, tf = use_cluster(n = i, frame=i, rotframes=360)<br>
<br>
tf.plot("%s_tf.png" % pre_name)<br>
<br>
tf.light_dir = (0.,0.,1.)<br>
tf.light_color = (0.0, 0.3, 0.3)<br>
tf.use_light = 0<br>
<br>
t1 = time.time()<br>
if os.path.isfile("%s_partitioned.h5" % pf):<br>
to_export = 0<br>
grids = import_partitioned_grids("%s_partitioned.h5" % pf)<br>
else:<br>
to_export = 1<br>
grids = None<br>
<br>
grids, image, vectors, norm_vec, pos = direct_ray_cast(<br>
pf, L, c, W, Nvec, tf, partitioned_grids=grids, whole_box=True)<br>
<br>
v = 0.0<br>
for g in grids: v += (g.RightEdge - g.LeftEdge).prod()<br>
print "VOLUME", v<br>
<br>
print pf, L, c, W, tf<br>
<br>
t2 = time.time()<br>
<br>
ma = image[:,:,:3].max()<br>
nan_image = na.zeros(image.shape, dtype='float64')<br>
nan_image[na.isnan(image)] = 1.0<br>
image[na.isnan(image)] = 0.0<br>
to_plot = image[:,:,:3]<br>
<br>
avgval = to_plot[to_plot>0].mean()<br>
stdval = to_plot[to_plot>0].std()<br>
maxval = to_plot[to_plot>0].max()<br>
print avgval, stdval, maxval, (avgval+stdval)/maxval<br>
<br>
alt_plot = (to_plot - to_plot.min()) / (avgval+5*stdval)<br>
<br>
to_plot = na.clip(to_plot, to_plot.min(), 0.8 * to_plot.max())<br>
to_plot = (to_plot - to_plot.min()) / (to_plot.max() - to_plot.min())<br>
<br>
to_plot[to_plot>1.0]=1.0<br>
alt_plot[alt_plot>1.0]=1.0<br>
<br>
print to_plot.max(), to_plot.min()<br>
print "%0.3e" % (t2-t1)<br>
<br>
pylab.clf()<br>
pylab.gcf().set_dpi(100)<br>
pylab.gcf().set_size_inches((Nvec/100.0, Nvec/100.0))<br>
pylab.gcf().subplots_adjust(left=0.0, right=1.0, bottom=0.0, top=1.0, wspace=0.0, hspace=0.0)<br>
pylab.imshow(alt_plot, interpolation='nearest')<br>
pylab.text(20, 20,'z = %0.3f' % pf['CosmologyCurrentRedshift'],<br>
color = 'w',size='large')<br>
pylab.savefig("%s_alt_rgb.png" % pre_name)<br>
pylab.clf()<br>
<br>
if to_export:<br>
export_partitioned_grids(grids, "%s_partitioned.h5" % pf)<br>
<br>
del grids, image, vectors, norm_vec, pos, ma, nan_image, to_plot, pf, L, c, W, tf<br>
-----------------------------------------------------------------------------------------------------------------------------------------<br>
<div><div></div><div>_______________________________________________<br>
yt-users mailing list<br>
<a href="mailto:yt-users@lists.spacepope.org" target="_blank">yt-users@lists.spacepope.org</a><br>
<a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
</div></div></blockquote></div><br></div>