<div dir="ltr">This is the full error if it helps at all? It is indeed, loading in all of the quantities.<div><br></div><div><div>Loading particles...</div><div> --> Loading particle type: 1</div><div>yt : [INFO ] 2014-06-08 15:12:05,540 Parameters: current_time = 0.0</div>
<div>yt : [INFO ] 2014-06-08 15:12:05,540 Parameters: domain_dimensions = [2 2 2]</div><div>yt : [INFO ] 2014-06-08 15:12:05,541 Parameters: domain_left_edge = [ 0. 0. 0.]</div><div>yt : [INFO ] 2014-06-08 15:12:05,542 Parameters: domain_right_edge = [ 100. 100. 100.]</div>
<div>yt : [INFO ] 2014-06-08 15:12:05,542 Parameters: cosmological_simulation = 0.0</div><div>yt : [INFO ] 2014-06-08 15:12:05,548 Allocating for 1.074e+09 particles</div><div>yt : [INFO ] 2014-06-08 15:16:16,195 Identified 7.584e+07 octs</div>
<div>yt : [INFO ] 2014-06-08 15:16:16,299 Loading field plugins.</div><div>yt : [INFO ] 2014-06-08 15:16:16,299 Loaded angular_momentum (8 new fields)</div><div>yt : [INFO ] 2014-06-08 15:16:16,299 Loaded astro (14 new fields)</div>
<div>yt : [INFO ] 2014-06-08 15:16:16,300 Loaded cosmology (20 new fields)</div><div>yt : [INFO ] 2014-06-08 15:16:16,300 Loaded fluid (56 new fields)</div><div>yt : [INFO ] 2014-06-08 15:16:16,301 Loaded fluid_vector (88 new fields)</div>
<div>yt : [INFO ] 2014-06-08 15:16:16,301 Loaded geometric (103 new fields)</div><div>yt : [INFO ] 2014-06-08 15:16:16,301 Loaded local (103 new fields)</div><div>yt : [INFO ] 2014-06-08 15:16:16,302 Loaded magnetic_field (109 new fields)</div>
<div>yt : [INFO ] 2014-06-08 15:16:16,302 Loaded species (109 new fields)</div><div>---------------------------------------------------------------------------</div><div>MemoryError Traceback (most recent call last)</div>
<div>/nfs/blank/h4231/bgriffen/data/lib/yt-x86_64/lib/python2.7/site-packages/IPython/utils/py3compat.pyc in execfile(fname, *where)</div><div> 202 else:</div><div> 203 filename = fname</div>
<div>--> 204 __builtin__.execfile(filename, *where)</div><div><br></div><div>/nfs/blank/h4231/bgriffen/work/projects/caterpillar/c2ray/cic/ytcic.py in <module>()</div><div> 99 slc.set_figure_size(4)</div>
<div> 100 slc.save()</div><div>--> 101 </div><div> 102 for ndim in dimlist:</div><div> 103 print</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in __getitem__(self, key)</div>
<div> 218 return self.field_data[f]</div><div> 219 else:</div><div>--> 220 self.get_data(f)</div><div> 221 # fi.units is the unit expression string. We depend on the registry</div>
<div> 222 # hanging off the dataset to define this unit object.</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in get_data(self, fields)</div><div> 627 </div>
<div> 628 fields_to_generate += gen_fluids + gen_particles</div><div>--> 629 self._generate_fields(fields_to_generate)</div><div> 630 </div><div> 631 def _generate_fields(self, fields_to_generate):</div>
<div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in _generate_fields(self, fields_to_generate)</div><div> 644 fi = self.pf._get_field_info(*field)</div>
<div> 645 try:</div><div>--> 646 fd = self._generate_field(field)</div><div> 647 if type(fd) == np.ndarray:</div><div> 648 fd = self.pf.arr(fd, fi.units)</div>
<div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in _generate_field(self, field)</div><div> 255 tr = self._generate_particle_field(field)</div><div>
256 else:</div><div>--> 257 tr = self._generate_fluid_field(field)</div><div> 258 if tr is None:</div><div> 259 raise YTCouldNotGenerateField(field, <a href="http://self.pf">self.pf</a>)</div>
<div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in _generate_fluid_field(self, field)</div><div> 273 finfo.check_available(gen_obj)</div><div> 274 except NeedsGridType as ngt_exception:</div>
<div>--> 275 rv = self._generate_spatial_fluid(field, ngt_exception.ghost_zones)</div><div> 276 else:</div><div> 277 rv = finfo(gen_obj)</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in _generate_spatial_fluid(self, field, ngz)</div>
<div> 289 o = self._current_chunk.objs[0]</div><div> 290 with o._activate_cache():</div><div>--> 291 ind += o.select(self.selector, self[field], rv, ind)</div>
<div> 292 else:</div><div> 293 chunks = self.index._chunk(self, "spatial", ngz = ngz)</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in __getitem__(self, key)</div>
<div> 218 return self.field_data[f]</div><div> 219 else:</div><div>--> 220 self.get_data(f)</div><div> 221 # fi.units is the unit expression string. We depend on the registry</div>
<div> 222 # hanging off the dataset to define this unit object.</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in get_data(self, fields)</div><div> 627 </div>
<div> 628 fields_to_generate += gen_fluids + gen_particles</div><div>--> 629 self._generate_fields(fields_to_generate)</div><div> 630 </div><div> 631 def _generate_fields(self, fields_to_generate):</div>
<div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in _generate_fields(self, fields_to_generate)</div><div> 644 fi = self.pf._get_field_info(*field)</div>
<div> 645 try:</div><div>--> 646 fd = self._generate_field(field)</div><div> 647 if type(fd) == np.ndarray:</div><div> 648 fd = self.pf.arr(fd, fi.units)</div>
<div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in _generate_field(self, field)</div><div> 255 tr = self._generate_particle_field(field)</div><div>
256 else:</div><div>--> 257 tr = self._generate_fluid_field(field)</div><div> 258 if tr is None:</div><div> 259 raise YTCouldNotGenerateField(field, <a href="http://self.pf">self.pf</a>)</div>
<div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc in _generate_fluid_field(self, field)</div><div> 275 rv = self._generate_spatial_fluid(field, ngt_exception.ghost_zones)</div>
<div> 276 else:</div><div>--> 277 rv = finfo(gen_obj)</div><div> 278 return rv</div><div> 279 </div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/fields/derived_field.pyc in __call__(self, data)</div>
<div> 176 "for %s" % (<a href="http://self.name">self.name</a>,))</div><div> 177 with self.unit_registry(data):</div><div>--> 178 dd = self._function(self, data)</div>
<div> 179 for field_name in data.keys():</div><div> 180 if field_name not in original_fields:</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/fields/particle_fields.pyc in particle_cic(field, data)</div>
<div> 113 def particle_cic(field, data):</div><div> 114 pos = data[ptype, coord_name]</div><div>--> 115 d = data.deposit(pos, [data[ptype, mass_name]], method = "cic")</div><div> 116 d = data.apply_units(d, data[ptype, mass_name].units)</div>
<div> 117 d /= data["index", "cell_volume"]</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/octree_subset.pyc in deposit(self, positions, fields, method)</div>
<div> 167 mylog.debug("Depositing %s (%s^3) particles into %s Octs",</div><div> 168 positions.shape[0], positions.shape[0]**0.3333333, nvals[-1])</div><div>--> 169 pos = np.asarray(positions.convert_to_units("code_length"),</div>
<div> 170 dtype="float64")</div><div> 171 # We should not need the following if we know in advance all our fields</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/units/yt_array.pyc in convert_to_units(self, units)</div>
<div> 366 </div><div> 367 self.units = new_units</div><div>--> 368 self *= conversion_factor</div><div> 369 return self</div><div> 370 </div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/units/yt_array.pyc in __imul__(self, other)</div>
<div> 667 """ See __mul__. """</div><div> 668 oth = sanitize_units_mul(self, other)</div><div>--> 669 return np.multiply(self, oth, out=self)</div><div> 670 </div>
<div> 671 def __div__(self, right_object):</div><div><br></div><div>/bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/units/yt_array.pyc in __array_wrap__(self, out_arr, context)</div><div> 966 # casting to YTArray avoids creating a YTQuantity with size > 1</div>
<div> 967 return YTArray(np.array(out_arr, unit))</div><div>--> 968 return ret_class(np.array(out_arr), unit)</div><div> 969 </div><div> 970 </div><div><br></div><div>MemoryError:</div>
</div></div><div class="gmail_extra"><br><br><div class="gmail_quote">On Sun, Jun 8, 2014 at 10:50 PM, Matthew Turk <span dir="ltr"><<a href="mailto:matthewturk@gmail.com" target="_blank">matthewturk@gmail.com</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Hi Brendan,<br>
<div class=""><br>
On Sun, Jun 8, 2014 at 9:21 PM, Brendan Griffen<br>
<<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>> wrote:<br>
> Hi Matt,<br>
><br>
> Thanks for your detailed email. Forgive my naivety but why do you need the<br>
> oct-tree in the first place? I have a my own fortran code for constructing a<br>
> cloud in cell mesh and it uses very little overhead (just the n^3 grid and<br>
> the particle data itself). I then calculate the dx,dy,dzs to the nearest 8<br>
> grid points and distribute accordingly in a omp loop which is done in a<br>
> fraction of a second. Does the situation with yt come about (oct tree etc.)<br>
> necessarily because of the way it handles particle data? Is it essentially<br>
> used to map the particles to domains in the grid or something?<br>
<br>
</div>That's not naive at all. There are two reasons --<br>
<br>
1) The octree is used for indexing for neighbor lookups and<br>
early-termination of region selection for particles<br>
2) The octree is used to estimate the "required resolution" for any<br>
operation that requires a space-filling value. (i.e., any time that a<br>
particle becomes a volume.)<br>
<br>
Projections in yt are adaptive, in that they project down to the<br>
finest appropriate resolution. There's also the "arbitrary_grid"<br>
operation, which does precisely what you're describing, but as it<br>
stands right now the octree gets constructed at time of instantiation<br>
of the indexing system. Thinking it over, you may be able to avoid<br>
that completely by not using load_particles and instead using<br>
load_uniform_grid and supplying your desired dimensions. The field<br>
names should be the same.<br>
<div class=""><br>
><br>
> The machine I max memory on has 128GB and the snapshots are using 1024^3<br>
> particles. Do you have any idea of how much memory the oct-tree uses as a<br>
> function of particle/grid number? I am going to try on a 256GB machine<br>
> (though this is a bit of a hassle). I'll see how I go.<br>
<br>
</div>I am disappointed that it's blowing out your RAM. This week I will<br>
try to get some memory profiling done. Could you file a bug to this<br>
effect, which will help me track it? Peak memory usage during<br>
indexing should only be 64 bits * Nparticles, unless you're using<br>
load_particles, in which case all the fields will *also* have to be in<br>
memory. It's about 8 gigabytes per field. So, I think there's<br>
something going wrong.<br>
<div class="HOEnZb"><div class="h5"><br>
><br>
> Thanks.<br>
><br>
> Brendan<br>
><br>
><br>
> On Sun, Jun 8, 2014 at 6:25 PM, Matthew Turk <<a href="mailto:matthewturk@gmail.com">matthewturk@gmail.com</a>> wrote:<br>
>><br>
>> Hi all,<br>
>><br>
>> I feel like I owe a brief explanation of why things are tricky right<br>
>> now, what we're planning on doing, and how we're experimenting and<br>
>> developing.<br>
>><br>
>> Presently, the particle geometry handlers build a single mesh from all<br>
>> particles in the dataset, along with a coarse bitmask that correlates<br>
>> files to regions in the domain. This requires the allocation of a<br>
>> single int64 array of size Nparticles, which is sorted in place and<br>
>> then fed into an octree construction algorithm that then spits back<br>
>> out the mesh. Each octree component contains 3 64-bit integers and<br>
>> eitehr a void pointer or a pointer to eight other octs. Increasing<br>
>> n_ref decreases the number of octs in this mesh; when smoothing<br>
>> operaitons are conducted, a second "index" mesh is created for looking<br>
>> up particles near mesh points. Mesh points are used for adaptive<br>
>> resolution smoothing and other "deposit particles on the grid somehow"<br>
>> operations (including SPH kernel).<br>
>><br>
>> Anyway, because right now it requires a global mesh to be constructed,<br>
>> this is expensive and requires holding a 64-bit integer in memory for<br>
>> each particle. I think if you're loading the particles in differently<br>
>> there is some additional overhead as well, but I'm still a bit<br>
>> surprised you OOM on a 1024^3 dataset.<br>
>><br>
>> In general, we don't *need* this global mesh; is can be constructed as<br>
>> required, which would speed up both the initial index phase as well as<br>
>> the final meshing process. I got about 50% of the way to implementing<br>
>> this last fall, but because of various concerns and deadlines I<br>
>> haven't finished it. I intend to get back to it probably in July,<br>
>> right after we put out a 3.0, so that we can have it in time for 3.1.<br>
>> In principle this will make the particle codes much more similar to<br>
>> ARTIO, in that the mesh will be constructed only as required and<br>
>> discarded when no longer required, which will make them much more<br>
>> memory efficient.<br>
>><br>
>> But, getting a single mesh for extremely large data is a very high<br>
>> priority; right now for the 10240^3 run we've been loading up<br>
>> individual sub-chunks, which I want to stop doing.<br>
>><br>
>> From the technical perspective, these are the things that need to<br>
>> happen on the yt side for particle datasets to move to this "lazy"<br>
>> mode of loading; most of this is based on things learned from 2HOT and<br>
>> ARTIO, and will involve converting to a forest-of-octrees.<br>
>><br>
>> * Split into spatially-organized subchunks of ParticleOctreeSubset<br>
>> objects, such that these map 1:Nfiles, and that can be constructed on<br>
>> the fly.<br>
>> * Construct a dual-mesh of the bitmask "ParticleRegion" object that<br>
>> will help with identifying neighbors to a given oct cell, so that if<br>
>> we're inside one octree we know which neighbor octrees to grab if we<br>
>> need particles for smoothing things (fast boundary particle<br>
>> identification is later down the road)<br>
>> * Parallel sort of particles, or using the parallel ring function;<br>
>> may not be necessary after all<br>
>><br>
>> All of this is doable, and I'd be happy to work with people if they'd<br>
>> like to take a shot at implementing it, but I've mostly put it on my<br>
>> list for post-3.0.<br>
>><br>
>> -Matt<br>
>><br>
>> On Sun, Jun 8, 2014 at 2:43 PM, Nathan Goldbaum <<a href="mailto:nathan12343@gmail.com">nathan12343@gmail.com</a>><br>
>> wrote:<br>
>> ><br>
>> ><br>
>> ><br>
>> > On Sun, Jun 8, 2014 at 12:27 PM, Brendan Griffen<br>
>> > <<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>> wrote:<br>
>> >><br>
>> >> Also, how do I construct just a zero filled yt array with dimensions<br>
>> >> (ndim,ndim,ndim)? Thanks<br>
>> ><br>
>> ><br>
>> ><br>
>> > from yt import YTArray<br>
>> > from numpy import np<br>
>> ><br>
>> > arr = YTArray(np.zeros([ndim, ndim, ndim]), input_units=units_string)<br>
>> ><br>
>> > or alternatively:<br>
>> ><br>
>> > from yt.units import kiloparsec<br>
>> ><br>
>> > arr = kiloparsec*np.zeros([ndim, ndim, ndim])<br>
>> ><br>
>> > it doesn't have to be kiloparsec - you can compose the units you want<br>
>> > out of<br>
>> > any of the unit symbols that live in yt.units.<br>
>> ><br>
>> > See this page for a ton more detail about yt's new unit system:<br>
>> > <a href="http://yt-project.org/docs/dev-3.0/analyzing/units/index.html" target="_blank">http://yt-project.org/docs/dev-3.0/analyzing/units/index.html</a><br>
>> ><br>
>> >><br>
>> >><br>
>> >> Brendan<br>
>> >><br>
>> >><br>
>> >> On Sun, Jun 8, 2014 at 3:26 PM, Brendan Griffen<br>
>> >> <<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>> wrote:<br>
>> >>><br>
>> >>> Hi,<br>
>> >>><br>
>> >>> Since I get memory errors. Could I not just read in the blocks of the<br>
>> >>> output individually then basically stack the mesh each time. That way<br>
>> >>> not<br>
>> >>> every single particle of the snapshot has to be loaded at the same<br>
>> >>> time.<br>
>> >>> Would that just be a case of doing<br>
>> >>><br>
>> >>> level = int(math.log(ndim,2))<br>
>> >>> cg = ds.covering_grid(level=level,<br>
>> >>> left_edge=[0,0,0],dims=[ndim,ndim,ndim])<br>
>> >>> arr = cg['deposit', 'all_density']<br>
>> >>> arrall += arr<br>
>> >>><br>
>> >>> in a loop over each HDF5 block?<br>
>> ><br>
>> ><br>
>> > It's likely that the memory use is dominated by the octree rather than<br>
>> > the<br>
>> > covering grid. This is with 1024^3 particles, correct?<br>
>> ><br>
>> > You can probably significantly reduce the memory used by the octree by<br>
>> > increasing n_ref in the call to load_particles.<br>
>> ><br>
>> > See this page for more detail about load_particles:<br>
>> ><br>
>> > <a href="http://yt-project.org/docs/dev-3.0/examining/loading_data.html#generic-particle-data" target="_blank">http://yt-project.org/docs/dev-3.0/examining/loading_data.html#generic-particle-data</a><br>
>> ><br>
>> > Larger n_ref means fewer octree cells (lower resolution), but it also<br>
>> > means<br>
>> > lower poisson noise and lower memory use.<br>
>> ><br>
>> > Alternatively, as Matt suggested, you could break your 1024^3 ensemble<br>
>> > of<br>
>> > particles up into chunks, loop over the chunk, creating a particle<br>
>> > octree<br>
>> > and then a covering grid for each subset of the particles. Your final<br>
>> > covering grid is just the sub of the covering grids for each subset of<br>
>> > particles.<br>
>> ><br>
>> >>><br>
>> >>><br>
>> >>> Thanks.<br>
>> >>> Brendan<br>
>> >>><br>
>> >>><br>
>> >>><br>
>> >>><br>
>> >>> On Fri, Jun 6, 2014 at 7:26 PM, Matthew Turk <<a href="mailto:matthewturk@gmail.com">matthewturk@gmail.com</a>><br>
>> >>> wrote:<br>
>> >>>><br>
>> >>>><br>
>> >>>> On Jun 6, 2014 4:54 PM, "Brendan Griffen"<br>
>> >>>> <<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>><br>
>> >>>> wrote:<br>
>> >>>> ><br>
>> >>>> > OK great. It is very low resolution but it worked. Thanks for all<br>
>> >>>> > your<br>
>> >>>> > help. My higher resolution run 1024^3 in 100 Mpc seems to crash on<br>
>> >>>> > 128GB<br>
>> >>>> > memory machine. I might have to look elsewhere.<br>
>> >>>> ><br>
>> >>>><br>
>> >>>> If you are looking for low resolution extraction you can tune the<br>
>> >>>> memory usage by changing the parameter n_ref to something higher.<br>
>> >>>><br>
>> >>>> Supporting extremely large datasets in a single mesh is on the<br>
>> >>>> roadmap<br>
>> >>>> for the late summer or fall, after a 3.0 release goes out. For now<br>
>> >>>> you can<br>
>> >>>> also extract before you load in; this is sort of how we are<br>
>> >>>> supporting an<br>
>> >>>> INCITE project with very large particle counts.<br>
>> >>>><br>
>> >>>><br>
>> >>>> > Also, I normally use Canopy distribution but I just use an alias to<br>
>> >>>> > loadyt which erases my PYTHONPATH and I can't access scipy and a<br>
>> >>>> > few other<br>
>> >>>> > libraries any more. What is the best practice here? Should I just<br>
>> >>>> > manually<br>
>> >>>> > export PYTHONPATH and point to the libraries need in canopy or can<br>
>> >>>> > they play<br>
>> >>>> > nice together?<br>
>> >>>> ><br>
>> >>>> > Thanks.<br>
>> >>>> ><br>
>> >>>> > BG<br>
>> >>>> ><br>
>> >>>> ><br>
>> >>>> > On Fri, Jun 6, 2014 at 2:54 PM, Nathan Goldbaum<br>
>> >>>> > <<a href="mailto:nathan12343@gmail.com">nathan12343@gmail.com</a>> wrote:<br>
>> >>>> >><br>
>> >>>> >><br>
>> >>>> >><br>
>> >>>> >><br>
>> >>>> >> On Fri, Jun 6, 2014 at 11:48 AM, Brendan Griffen<br>
>> >>>> >> <<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>> wrote:<br>
>> >>>> >>><br>
>> >>>> >>> OK great. Thanks. I just wanted a homogeneous mesh. 512^3 with no<br>
>> >>>> >>> nesting of any kind. Though when I plot the image it looks like<br>
>> >>>> >>> it is<br>
>> >>>> >>> assigning particles incorrectly (low resolution on the outside).<br>
>> >>>> >>> This is<br>
>> >>>> >>> just a test image.<br>
>> >>>> >>><br>
>> >>>> >><br>
>> >>>> >> The SlicePlot is visualizing the octree so there is less<br>
>> >>>> >> resolution<br>
>> >>>> >> where there are fewer particles. If you want to visualize the<br>
>> >>>> >> covering grid<br>
>> >>>> >> you're going to need to visualize that separately.<br>
>> >>>> >><br>
>> >>>> >>><br>
>> >>>> >>> ds = yt.load_particles(data, length_unit=3.08e24,<br>
>> >>>> >>> mass_unit=1.9891e33,bbox=bbox)<br>
>> >>>> >>><br>
>> >>>> >>> ad = ds.all_data()<br>
>> >>>> >>> print ad['deposit', 'all_cic']<br>
>> >>>> >>> slc = yt.SlicePlot(ds, 2, ('deposit', 'all_cic'))<br>
>> >>>> >>> slc.set_figure_size(4)<br>
>> >>>> >>> cg = ds.covering_grid(level=9,<br>
>> >>>> >>> left_edge=[0,0,0],dims=[512,512,512])<br>
>> >>>> >>><br>
>> >>>> >><br>
>> >>>> >> To actually produce the uniform resolution ndarray, you're going<br>
>> >>>> >> to<br>
>> >>>> >> need to do something like:<br>
>> >>>> >><br>
>> >>>> >> array = cg[('deposit', 'all_cic')]<br>
>> >>>> >><br>
>> >>>> >> array will then be a 3D array you can do whatever you want with.<br>
>> >>>> >> By<br>
>> >>>> >> default it has units, but to strip them off you'll just need to<br>
>> >>>> >> cast to<br>
>> >>>> >> ndarray:<br>
>> >>>> >><br>
>> >>>> >> array_without_units = array.v<br>
>> >>>> >><br>
>> >>>> >><br>
>> >>>> >>><br>
>> >>>> >>> Also, is there a way to load multiple particle types?<br>
>> >>>> >>><br>
>> >>>> >>> Do I just need to stack the particles into the array here?<br>
>> >>>> >>><br>
>> >>>> >>> data = {'particle_position_x': pos[:,0],<br>
>> >>>> >>> 'particle_position_y': pos[:,1],<br>
>> >>>> >>> 'particle_position_z': pos[:,2],<br>
>> >>>> >>> 'particle_mass': np.array([mpart]*npart)}<br>
>> >>>> >>><br>
>> >>>> >>> Then feed it in as usual?<br>
>> >>>> >><br>
>> >>>> >><br>
>> >>>> >> That's right, although if the particle masses are different for<br>
>> >>>> >> the<br>
>> >>>> >> different particle types that code snippet will need to be<br>
>> >>>> >> generalized to<br>
>> >>>> >> handle that.<br>
>> >>>> >><br>
>> >>>> >> I think in principle it should be possible to make load_particles<br>
>> >>>> >> handle different particle types just like an SPH dataset that<br>
>> >>>> >> contains<br>
>> >>>> >> multiple particle types, but right now that hasn't been<br>
>> >>>> >> implemented yet.<br>
>> >>>> >><br>
>> >>>> >>><br>
>> >>>> >>><br>
>> >>>> >>> Brendan<br>
>> >>>> >>><br>
>> >>>> >>><br>
>> >>>> >>> On Thu, Jun 5, 2014 at 9:44 PM, Nathan Goldbaum<br>
>> >>>> >>> <<a href="mailto:nathan12343@gmail.com">nathan12343@gmail.com</a>> wrote:<br>
>> >>>> >>>><br>
>> >>>> >>>> That's right, you can set that via the bbox keyword parameter<br>
>> >>>> >>>> for<br>
>> >>>> >>>> load_particles. I'd urge you to take a look at the docstrings<br>
>> >>>> >>>> and source<br>
>> >>>> >>>> code for load_particles.<br>
>> >>>> >>>><br>
>> >>>> >>>><br>
>> >>>> >>>> On Thu, Jun 5, 2014 at 6:34 PM, Brendan Griffen<br>
>> >>>> >>>> <<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>> wrote:<br>
>> >>>> >>>>><br>
>> >>>> >>>>> Thanks very much Nathan. I tried to load in my own data but I<br>
>> >>>> >>>>> think there are too many particles or I have to specifically<br>
>> >>>> >>>>> set the domain<br>
>> >>>> >>>>> size.<br>
>> >>>> >>>>><br>
>> >>>> >>>>> In this area:<br>
>> >>>> >>>>><br>
>> >>>> >>>>> data = {'particle_position_x': pos[:,0],<br>
>> >>>> >>>>> 'particle_position_y': pos[:,1],<br>
>> >>>> >>>>> 'particle_position_z': pos[:,2],<br>
>> >>>> >>>>> 'particle_mass': np.array([mpart]*npart)}<br>
>> >>>> >>>>><br>
>> >>>> >>>>> ds = yt.load_particles(data, length_unit=3.08e24,<br>
>> >>>> >>>>> mass_unit=1.9891e36)<br>
>> >>>> >>>>> ad = ds.all_data()<br>
>> >>>> >>>>> print ad['deposit', 'all_cic']<br>
>> >>>> >>>>><br>
>> >>>> >>>>> In [3]: run ytcic.py<br>
>> >>>> >>>>> yt : [INFO ] 2014-06-05 21:29:06,183 Parameters:<br>
>> >>>> >>>>> current_time<br>
>> >>>> >>>>> = 0.0<br>
>> >>>> >>>>> yt : [INFO ] 2014-06-05 21:29:06,183 Parameters:<br>
>> >>>> >>>>> domain_dimensions = [2 2 2]<br>
>> >>>> >>>>> yt : [INFO ] 2014-06-05 21:29:06,184 Parameters:<br>
>> >>>> >>>>> domain_left_edge = [ 0. 0. 0.]<br>
>> >>>> >>>>> yt : [INFO ] 2014-06-05 21:29:06,185 Parameters:<br>
>> >>>> >>>>> domain_right_edge = [ 1. 1. 1.]<br>
>> >>>> >>>>> yt : [INFO ] 2014-06-05 21:29:06,185 Parameters:<br>
>> >>>> >>>>> cosmological_simulation = 0.0<br>
>> >>>> >>>>> yt : [INFO ] 2014-06-05 21:29:06,188 Allocating for<br>
>> >>>> >>>>> 1.342e+08<br>
>> >>>> >>>>> particles<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> ---------------------------------------------------------------------------<br>
>> >>>> >>>>> YTDomainOverflow Traceback (most<br>
>> >>>> >>>>> recent<br>
>> >>>> >>>>> call last)<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /nfs/blank/h4231/bgriffen/data/lib/yt-x86_64/lib/python2.7/site-packages/IPython/utils/py3compat.pyc<br>
>> >>>> >>>>> in execfile(fname, *where)<br>
>> >>>> >>>>> 202 else:<br>
>> >>>> >>>>> 203 filename = fname<br>
>> >>>> >>>>> --> 204 __builtin__.execfile(filename, *where)<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /nfs/blank/h4231/bgriffen/work/projects/caterpillar/c2ray/cic/ytcic.py in<br>
>> >>>> >>>>> <module>()<br>
>> >>>> >>>>> 52<br>
>> >>>> >>>>> 53 ad = ds.all_data()<br>
>> >>>> >>>>> ---> 54 print ad['deposit', 'all_cic']<br>
>> >>>> >>>>> 55 slc = yt.SlicePlot(ds, 2, ('deposit', 'all_cic'))<br>
>> >>>> >>>>> 56 slc.set_figure_size(4)<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc<br>
>> >>>> >>>>> in __getitem__(self, key)<br>
>> >>>> >>>>> 205 Returns a single field. Will add if necessary.<br>
>> >>>> >>>>> 206 """<br>
>> >>>> >>>>> --> 207 f = self._determine_fields([key])[0]<br>
>> >>>> >>>>> 208 if f not in self.field_data and key not in<br>
>> >>>> >>>>> self.field_data:<br>
>> >>>> >>>>> 209 if f in self._container_fields:<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/data_containers.pyc<br>
>> >>>> >>>>> in _determine_fields(self, fields)<br>
>> >>>> >>>>> 453 raise YTFieldNotParseable(field)<br>
>> >>>> >>>>> 454 ftype, fname = field<br>
>> >>>> >>>>> --> 455 finfo = self.pf._get_field_info(ftype,<br>
>> >>>> >>>>> fname)<br>
>> >>>> >>>>> 456 else:<br>
>> >>>> >>>>> 457 fname = field<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/static_output.pyc<br>
>> >>>> >>>>> in _get_field_info(self, ftype, fname)<br>
>> >>>> >>>>> 445 _last_finfo = None<br>
>> >>>> >>>>> 446 def _get_field_info(self, ftype, fname = None):<br>
>> >>>> >>>>> --> 447 self.index<br>
>> >>>> >>>>> 448 if fname is None:<br>
>> >>>> >>>>> 449 ftype, fname = "unknown", ftype<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/data_objects/static_output.pyc<br>
>> >>>> >>>>> in index(self)<br>
>> >>>> >>>>> 277 raise RuntimeError("You should not<br>
>> >>>> >>>>> instantiate Dataset.")<br>
>> >>>> >>>>> 278 self._instantiated_index =<br>
>> >>>> >>>>> self._index_class(<br>
>> >>>> >>>>> --> 279 self, dataset_type=self.dataset_type)<br>
>> >>>> >>>>> 280 # Now we do things that we need an<br>
>> >>>> >>>>> instantiated index for<br>
>> >>>> >>>>> 281 # ...first off, we create our field_info<br>
>> >>>> >>>>> now.<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/frontends/stream/data_structures.pyc<br>
>> >>>> >>>>> in __init__(self, pf, dataset_type)<br>
>> >>>> >>>>> 942 def __init__(self, pf, dataset_type = None):<br>
>> >>>> >>>>> 943 self.stream_handler = pf.stream_handler<br>
>> >>>> >>>>> --> 944 super(StreamParticleIndex, self).__init__(pf,<br>
>> >>>> >>>>> dataset_type)<br>
>> >>>> >>>>> 945<br>
>> >>>> >>>>> 946 def _setup_data_io(self):<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/geometry/particle_geometry_handler.pyc<br>
>> >>>> >>>>> in __init__(self, pf, dataset_type)<br>
>> >>>> >>>>> 48 self.directory =<br>
>> >>>> >>>>> os.path.dirname(self.index_filename)<br>
>> >>>> >>>>> 49 self.float_type = np.float64<br>
>> >>>> >>>>> ---> 50 super(ParticleIndex, self).__init__(pf,<br>
>> >>>> >>>>> dataset_type)<br>
>> >>>> >>>>> 51<br>
>> >>>> >>>>> 52 def _setup_geometry(self):<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/geometry/geometry_handler.pyc<br>
>> >>>> >>>>> in __init__(self, pf, dataset_type)<br>
>> >>>> >>>>> 54<br>
>> >>>> >>>>> 55 mylog.debug("Setting up domain geometry.")<br>
>> >>>> >>>>> ---> 56 self._setup_geometry()<br>
>> >>>> >>>>> 57<br>
>> >>>> >>>>> 58 mylog.debug("Initializing data grid data IO")<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/geometry/particle_geometry_handler.pyc<br>
>> >>>> >>>>> in _setup_geometry(self)<br>
>> >>>> >>>>> 52 def _setup_geometry(self):<br>
>> >>>> >>>>> 53 mylog.debug("Initializing Particle Geometry<br>
>> >>>> >>>>> Handler.")<br>
>> >>>> >>>>> ---> 54 self._initialize_particle_handler()<br>
>> >>>> >>>>> 55<br>
>> >>>> >>>>> 56<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/geometry/particle_geometry_handler.pyc<br>
>> >>>> >>>>> in _initialize_particle_handler(self)<br>
>> >>>> >>>>> 87 pf.domain_left_edge,<br>
>> >>>> >>>>> pf.domain_right_edge,<br>
>> >>>> >>>>> 88 [N, N, N], len(self.data_files))<br>
>> >>>> >>>>> ---> 89 self._initialize_indices()<br>
>> >>>> >>>>> 90 self.oct_handler.finalize()<br>
>> >>>> >>>>> 91 self.max_level = self.oct_handler.max_level<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/geometry/particle_geometry_handler.pyc<br>
>> >>>> >>>>> in _initialize_indices(self)<br>
>> >>>> >>>>> 109 npart =<br>
>> >>>> >>>>> sum(data_file.total_particles.values())<br>
>> >>>> >>>>> 110 morton[ind:ind + npart] = \<br>
>> >>>> >>>>> --> 111 self.io._initialize_index(data_file,<br>
>> >>>> >>>>> self.regions)<br>
>> >>>> >>>>> 112 ind += npart<br>
>> >>>> >>>>> 113 morton.sort()<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> /bigbang/data/bgriffen/lib/yt-x86_64/src/yt-hg/yt/frontends/stream/io.pyc in<br>
>> >>>> >>>>> _initialize_index(self, data_file, regions)<br>
>> >>>> >>>>> 144 raise YTDomainOverflow(pos.min(axis=0),<br>
>> >>>> >>>>> pos.max(axis=0),<br>
>> >>>> >>>>> 145<br>
>> >>>> >>>>> data_file.pf.domain_left_edge,<br>
>> >>>> >>>>> --> 146<br>
>> >>>> >>>>> data_file.pf.domain_right_edge)<br>
>> >>>> >>>>> 147 regions.add_data_file(pos,<br>
>> >>>> >>>>> data_file.file_id)<br>
>> >>>> >>>>> 148 morton.append(compute_morton(<br>
>> >>>> >>>>><br>
>> >>>> >>>>> YTDomainOverflow: Particle bounds [ 0. 0. 0.] and [<br>
>> >>>> >>>>> 99.99999237<br>
>> >>>> >>>>> 99.99999237 99.99999237] exceed domain bounds [ 0. 0. 0.]<br>
>> >>>> >>>>> code_length and<br>
>> >>>> >>>>> [ 1. 1. 1.] code_length<br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> On Thu, Jun 5, 2014 at 8:22 PM, Nathan Goldbaum<br>
>> >>>> >>>>> <<a href="mailto:nathan12343@gmail.com">nathan12343@gmail.com</a>> wrote:<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> Here's a worked out example that does what you're looking for<br>
>> >>>> >>>>>> using a fake 1 million particle dataset:<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> <a href="http://nbviewer.ipython.org/gist/ngoldbaum/546d37869aafe71cfe38" target="_blank">http://nbviewer.ipython.org/gist/ngoldbaum/546d37869aafe71cfe38</a><br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> In this notebook I make use of two key yt features:<br>
>> >>>> >>>>>> `load_particles`, and `covering_grid`.<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> load_particles creates a "stream" dataset based on in-memory<br>
>> >>>> >>>>>> data<br>
>> >>>> >>>>>> fed in as a numpy array. This dataset acts just like an<br>
>> >>>> >>>>>> on-disk simulation<br>
>> >>>> >>>>>> dataset, but doesn't come with the baggage of needing to write<br>
>> >>>> >>>>>> a custom<br>
>> >>>> >>>>>> frontend to read a specific data format off disk.<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> covering_grid is a way to generate uniform resolution data<br>
>> >>>> >>>>>> from<br>
>> >>>> >>>>>> an AMR dataset. It acts like a python dictionary where the<br>
>> >>>> >>>>>> keys are field<br>
>> >>>> >>>>>> names and returns 3D numpy arrays of whatever uniform<br>
>> >>>> >>>>>> resolution you specify<br>
>> >>>> >>>>>> when you create the covering_grid.<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> Note that if you're using load_particles all of your data<br>
>> >>>> >>>>>> needs<br>
>> >>>> >>>>>> to live in memory. If your data is too big for that you'll<br>
>> >>>> >>>>>> need to write a<br>
>> >>>> >>>>>> frontend for your data format or use a memmap to an on-disk<br>
>> >>>> >>>>>> file somehow.<br>
>> >>>> >>>>>> I'm not an expert on that but others on the list should be<br>
>> >>>> >>>>>> able to help out.<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> Hope that gets you well on your way :)<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> -Nathan<br>
>> >>>> >>>>>><br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> On Thu, Jun 5, 2014 at 5:04 PM, Desika Narayanan<br>
>> >>>> >>>>>> <<a href="mailto:dnarayan@haverford.edu">dnarayan@haverford.edu</a>> wrote:<br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> Hey Brendan,<br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> A couple of extra tools you might find helpful in conjunction<br>
>> >>>> >>>>>>> with Nathan's example of depositing the particles onto an<br>
>> >>>> >>>>>>> octree are at:<br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> <a href="http://paste.yt-project.org/show/4737/" target="_blank">http://paste.yt-project.org/show/4737/</a><br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> Where I load a gadget snapshot, and then recover the<br>
>> >>>> >>>>>>> coordinates<br>
>> >>>> >>>>>>> and width of each cell.<br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> In response to your last question - the particles are<br>
>> >>>> >>>>>>> deposited<br>
>> >>>> >>>>>>> into an octree grid (so, you'll see that the cell sizes<br>
>> >>>> >>>>>>> aren't all the same<br>
>> >>>> >>>>>>> size). I don't know if depositing onto a regular NxNxN mesh<br>
>> >>>> >>>>>>> is possible,<br>
>> >>>> >>>>>>> though would be interested to hear if so.<br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> -d<br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> On Thu, Jun 5, 2014 at 7:58 PM, Brendan Griffen<br>
>> >>>> >>>>>>> <<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>> wrote:<br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>>> Thanks. I'll get the "bleeding edge" version first then try<br>
>> >>>> >>>>>>>> your suggestions. Though I want to return the NxNxN array<br>
>> >>>> >>>>>>>> and be able to<br>
>> >>>> >>>>>>>> write this mesh to a file. It is *only* using the cic part<br>
>> >>>> >>>>>>>> of yt and it<br>
>> >>>> >>>>>>>> should return the mesh to be written? Just wanted to<br>
>> >>>> >>>>>>>> clarify?<br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>>> Thanks.<br>
>> >>>> >>>>>>>> Brendan<br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>>> On Thu, Jun 5, 2014 at 6:49 PM, Nathan Goldbaum<br>
>> >>>> >>>>>>>> <<a href="mailto:nathan12343@gmail.com">nathan12343@gmail.com</a>> wrote:<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> On Thu, Jun 5, 2014 at 3:36 PM, John ZuHone<br>
>> >>>> >>>>>>>>> <<a href="mailto:jzuhone@gmail.com">jzuhone@gmail.com</a>> wrote:<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> Hi Brendan,<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> Which version of yt are you using?<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> If you're using 3.0, this is actually fairly easy. If you<br>
>> >>>> >>>>>>>>>> look in <a href="http://yt.fields.particle_fields.py" target="_blank">yt.fields.particle_fields.py</a>, around line 85, you<br>
>> >>>> >>>>>>>>>> can see how this<br>
>> >>>> >>>>>>>>>> is done for the "particle_density" and "particle_mass"<br>
>> >>>> >>>>>>>>>> fields. Basically you<br>
>> >>>> >>>>>>>>>> can call a "deposit" method which takes the particle field<br>
>> >>>> >>>>>>>>>> quantity you want<br>
>> >>>> >>>>>>>>>> deposited and deposits it into cells. The underlying<br>
>> >>>> >>>>>>>>>> calculation is done<br>
>> >>>> >>>>>>>>>> using Cython, so it's fast.<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> And you shouldn't ever actually need to call these<br>
>> >>>> >>>>>>>>> "deposit"<br>
>> >>>> >>>>>>>>> functions, since "deposit" is exposed as a field type for<br>
>> >>>> >>>>>>>>> all datasets that<br>
>> >>>> >>>>>>>>> contain particles.<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> Here is a notebook that does this for Enzo AMR data:<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> <a href="http://nbviewer.ipython.org/gist/ngoldbaum/5e19e4e6cc2bf330149c" target="_blank">http://nbviewer.ipython.org/gist/ngoldbaum/5e19e4e6cc2bf330149c</a><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> This dataset contains about a million particles and<br>
>> >>>> >>>>>>>>> generates<br>
>> >>>> >>>>>>>>> a CIC deposition for the whole domain in about 6 seconds<br>
>> >>>> >>>>>>>>> from a cold start.<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> If you're using 2.x, then you can do the same thing, but<br>
>> >>>> >>>>>>>>>> it's<br>
>> >>>> >>>>>>>>>> not as straightforward. You can see how this works in<br>
>> >>>> >>>>>>>>>> <a href="http://yt.data_objects.universal_fields.py" target="_blank">yt.data_objects.universal_fields.py</a>, around line 986,<br>
>> >>>> >>>>>>>>>> where the<br>
>> >>>> >>>>>>>>>> "particle_density" field is defined. Basically, it calls<br>
>> >>>> >>>>>>>>>> CICDeposit_3, which<br>
>> >>>> >>>>>>>>>> is in yt.utilities.lib.CICDeposit.pyx.<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> Let me know if you need any more clarification.<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> Best,<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> John Z<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> On Jun 5, 2014, at 6:07 PM, Brendan Griffen<br>
>> >>>> >>>>>>>>>> <<a href="mailto:brendan.f.griffen@gmail.com">brendan.f.griffen@gmail.com</a>> wrote:<br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> > Hi,<br>
>> >>>> >>>>>>>>>> ><br>
>> >>>> >>>>>>>>>> > I was wondering if there were any Cython routines within<br>
>> >>>> >>>>>>>>>> > yt<br>
>> >>>> >>>>>>>>>> > which takes particle data and converts it into a<br>
>> >>>> >>>>>>>>>> > cloud-in-cell based mesh<br>
>> >>>> >>>>>>>>>> > which can be written to a file of my choosing.<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> What sort of mesh were you looking for? yt will internally<br>
>> >>>> >>>>>>>>> construct an octree if it is fed particle data. I'm not<br>
>> >>>> >>>>>>>>> sure whether this<br>
>> >>>> >>>>>>>>> octree can be saved to disk for later analysis.<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> It's also possible to create a uniform resolution covering<br>
>> >>>> >>>>>>>>> grid containing field data for a deposited quantity, which<br>
>> >>>> >>>>>>>>> can be quite<br>
>> >>>> >>>>>>>>> easily saved to disk in a number of ways.<br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> I heard a while ago there was some such functionality but<br>
>> >>>> >>>>>>>>>> it<br>
>> >>>> >>>>>>>>>> could be too far down the yt rabbit hole to be used as a<br>
>> >>>> >>>>>>>>>> standalone? Is this<br>
>> >>>> >>>>>>>>>> true? I have my own Python code for doing it but it just<br>
>> >>>> >>>>>>>>>> isn't fast enough<br>
>> >>>> >>>>>>>>>> and thought I'd ask the yt community if there were any<br>
>> >>>> >>>>>>>>>> wrapper tools<br>
>> >>>> >>>>>>>>>> available to boost the speed.<br>
>> >>>> >>>>>>>>>> ><br>
>> >>>> >>>>>>>>>> > Thanks.<br>
>> >>>> >>>>>>>>>> > Brendan<br>
>> >>>> >>>>>>>>>> > _______________________________________________<br>
>> >>>> >>>>>>>>>> > yt-users mailing list<br>
>> >>>> >>>>>>>>>> > <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>>>>>>>> ><br>
>> >>>> >>>>>>>>>> ><br>
>> >>>> >>>>>>>>>> > <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> _______________________________________________<br>
>> >>>> >>>>>>>>>> yt-users mailing list<br>
>> >>>> >>>>>>>>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>><br>
>> >>>> >>>>>>>>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> _______________________________________________<br>
>> >>>> >>>>>>>>> yt-users mailing list<br>
>> >>>> >>>>>>>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>>>>>>><br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>>> _______________________________________________<br>
>> >>>> >>>>>>>> yt-users mailing list<br>
>> >>>> >>>>>>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>>>>>><br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> _______________________________________________<br>
>> >>>> >>>>>>> yt-users mailing list<br>
>> >>>> >>>>>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>>>>><br>
>> >>>> >>>>>><br>
>> >>>> >>>>>><br>
>> >>>> >>>>>> _______________________________________________<br>
>> >>>> >>>>>> yt-users mailing list<br>
>> >>>> >>>>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>><br>
>> >>>> >>>>> _______________________________________________<br>
>> >>>> >>>>> yt-users mailing list<br>
>> >>>> >>>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>>><br>
>> >>>> >>>><br>
>> >>>> >>>><br>
>> >>>> >>>> _______________________________________________<br>
>> >>>> >>>> yt-users mailing list<br>
>> >>>> >>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>>><br>
>> >>>> >>><br>
>> >>>> >>><br>
>> >>>> >>> _______________________________________________<br>
>> >>>> >>> yt-users mailing list<br>
>> >>>> >>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >>><br>
>> >>>> >><br>
>> >>>> >><br>
>> >>>> >> _______________________________________________<br>
>> >>>> >> yt-users mailing list<br>
>> >>>> >> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> >> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> >><br>
>> >>>> ><br>
>> >>>> ><br>
>> >>>> > _______________________________________________<br>
>> >>>> > yt-users mailing list<br>
>> >>>> > <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> > <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>> ><br>
>> >>>><br>
>> >>>><br>
>> >>>> _______________________________________________<br>
>> >>>> yt-users mailing list<br>
>> >>>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >>>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >>>><br>
>> >>><br>
>> >><br>
>> >><br>
>> >> _______________________________________________<br>
>> >> yt-users mailing list<br>
>> >> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> >> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> >><br>
>> ><br>
>> ><br>
>> > _______________________________________________<br>
>> > yt-users mailing list<br>
>> > <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> > <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
>> ><br>
>> _______________________________________________<br>
>> yt-users mailing list<br>
>> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
>> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
><br>
><br>
><br>
> _______________________________________________<br>
> yt-users mailing list<br>
> <a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
> <a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
><br>
_______________________________________________<br>
yt-users mailing list<br>
<a href="mailto:yt-users@lists.spacepope.org">yt-users@lists.spacepope.org</a><br>
<a href="http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org" target="_blank">http://lists.spacepope.org/listinfo.cgi/yt-users-spacepope.org</a><br>
</div></div></blockquote></div><br></div>