[yt-users] Issue loading large RAMSES dataset

[Søren Frimann]

Dear yt users,

I have an issue when I'm trying to load a RAMSES snapshot, which I think comes about because the snapshot is too large (51GB, code below). I have been able to successfully load several smaller simulation with yt.

Is there any way to circumvent this issue, for example by just loading a small part of the simulation into yt? I have tried to use some data objects to pick out regions of interest (ds.sphere, ds.box) but to no avail.

I have also considered just reading out the raw data from a region of interest like I usually do, and then read it into yt as a generic mesh, but it appears that you cannot do this in yt for an oct-tree AMR, only layered AMR.

If anyone have any insight as to how I might process I would be very grateful.

I use yt ver 3.2.3

Best,
Søren

#####################################################
#####################################################
#####################################################
import yt

ds = yt.load('./output_00400/info_00400.txt')
yt : [INFO     ] 2016-04-15 16:18:45,575 Parameters: current_time              = 403509140194502.0

yt : [INFO     ] 2016-04-15 16:18:45,575 Parameters: domain_dimensions         = [512 512 512]
yt : [INFO     ] 2016-04-15 16:18:45,576 Parameters: domain_left_edge          = [ 0.  0.  0.]
yt : [INFO     ] 2016-04-15 16:18:45,577 Parameters: domain_right_edge         = [ 1.  1.  1.]
yt : [INFO     ] 2016-04-15 16:18:45,577 Parameters: cosmological_simulation   = 0

ds.index

---------------------------------------------------------------------------
error                                     Traceback (most recent call last)
<ipython-input-3-2b95fa135b88> in <module>()
----> 1 ds.index

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/data_objects/static_output.py in index(self)
    319                 raise RuntimeError("You should not instantiate Dataset.")
    320             self._instantiated_index = self._index_class(
--> 321                 self, dataset_type=self.dataset_type)
    322             # Now we do things that we need an instantiated index for
    323             # ...first off, we create our field_info now.

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/frontends/ramses/data_structures.py in __init__(self, ds, dataset_type)
    349 
    350         self.float_type = np.float64
--> 351         super(RAMSESIndex, self).__init__(ds, dataset_type)
    352 
    353     def _initialize_oct_handler(self):

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/geometry/geometry_handler.py in __init__(self, ds, dataset_type)
     55 
     56         mylog.debug("Setting up domain geometry.")
---> 57         self._setup_geometry()
     58 
     59         mylog.debug("Initializing data grid data IO")

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/geometry/oct_geometry_handler.py in _setup_geometry(self)
     41     def _setup_geometry(self):
     42         mylog.debug("Initializing Octree Geometry Handler.")
---> 43         self._initialize_oct_handler()
     44 
     45     def get_smallest_dx(self):

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/frontends/ramses/data_structures.py in _initialize_oct_handler(self)
    353     def _initialize_oct_handler(self):
    354         self.domains = [RAMSESDomainFile(self.dataset, i + 1)
--> 355                         for i in range(self.dataset['ncpu'])]
    356         total_octs = sum(dom.local_oct_count #+ dom.ngridbound.sum()
    357                          for dom in self.domains)

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/frontends/ramses/data_structures.py in <listcomp>(.0)
    353     def _initialize_oct_handler(self):
    354         self.domains = [RAMSESDomainFile(self.dataset, i + 1)
--> 355                         for i in range(self.dataset['ncpu'])]
    356         total_octs = sum(dom.local_oct_count #+ dom.ngridbound.sum()
    357                          for dom in self.domains)

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/frontends/ramses/data_structures.py in __init__(self, ds, domain_id)
     62         for t in ['grav', 'hydro', 'part', 'amr']:
     63             setattr(self, "%s_fn" % t, basename % t)
---> 64         self._read_amr_header()
     65         self._read_hydro_header()
     66         self._read_particle_header()

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/frontends/ramses/data_structures.py in _read_amr_header(self)
    183         f = open(self.amr_fn, "rb")
    184         for header in ramses_header(hvals):
--> 185             hvals.update(fpu.read_attrs(f, header))
    186         # That's the header, now we skip a few.
    187         hvals['numbl'] = np.array(hvals['numbl']).reshape(

/astro/sfrimann/miniconda3/lib/python3.5/site-packages/yt/utilities/fortran_utils.py in read_attrs(f, attrs, endian)
     72         net_format += "".join(["I"] + ([t] * n) + ["I"])
     73     size = struct.calcsize(net_format)
---> 74     vals = list(struct.unpack(net_format, f.read(size)))
     75     vv = {}
     76     for a, n, t in attrs:

error: unpack requires a bytes object of length 61253615840
#####################################################
#####################################################
#####################################################

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.spacepope.org/pipermail/yt-users-spacepope.org/attachments/20160415/30ef5a88/attachment.htm>