I'm trying to save a panel that has a mixed type minor axis using HDFStore. I can save the panel when using the fixed format (although I get a PerformanceWarning), but it fails for the table format. Example code:
import pandas as pd
import numpy as np
d1 = pd.DataFrame({1:pd.Series(np.random.randn(10)), 2:pd.Series(np.random.randn(10))})
d2 = pd.DataFrame({1:pd.Series(np.random.randn(10)), 2:pd.Series(np.random.randn(10))})
good_panel = pd.Panel({'a':d1, 'b':d2})
print good_panel
with pd.get_store('/tmp/goodstore.h5', mode='w') as store:
store.put('/test', good_panel, 'table', append=False)
b1 = pd.DataFrame({1:pd.Series(np.random.randn(10)), 'a':pd.Series(np.random.randn(10))})
b2 = pd.DataFrame({1:pd.Series(np.random.randn(10)), 'a':pd.Series(np.random.randn(10))})
bad_panel = pd.Panel({'a':b1, 'b':b2})
print bad_panel
with pd.get_store('/tmp/badstore.h5', mode='w') as store:
store.put('/test', bad_panel, 'fixed', append=False)
print 'saved as fixed'
with pd.get_store('/tmp/badstore.h5', mode='w') as store:
store.put('/test', bad_panel, 'table', append=False)
The error I get is:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-5-3528d8319daf> in <module>()
9
10 with pd.get_store('/tmp/badstore.h5', mode='w') as store:
---> 11 store.put('/test', bad_panel, 'table', append=False)
/home/tony/.virtualenvs/dev2/lib/python2.7/site-packages/pandas/io/pytables.pyc in put(self, key, value, format, append, **kwargs)
829 format = get_option("io.hdf.default_format") or 'fixed'
830 kwargs = self._validate_format(format, kwargs)
--> 831 self._write_to_group(key, value, append=append, **kwargs)
832
833 def remove(self, key, where=None, start=None, stop=None):
/home/tony/.virtualenvs/dev2/lib/python2.7/site-packages/pandas/io/pytables.pyc in _write_to_group(self, key, value, format, index, append, complib, encoding, **kwargs)
1278
1279 # write the object
-> 1280 s.write(obj=value, append=append, complib=complib, **kwargs)
1281
1282 if s.is_table and index:
/home/tony/.virtualenvs/dev2/lib/python2.7/site-packages/pandas/io/pytables.pyc in write(self, obj, axes, append, complib, complevel, fletcher32, min_itemsize, chunksize, expectedrows, dropna, **kwargs)
3657
3658 # create the table
-> 3659 table = self._handle.create_table(self.group, **options)
3660
3661 else:
/home/tony/.virtualenvs/dev2/lib/python2.7/site-packages/tables/file.pyc in create_table(self, where, name, description, title, filters, expectedrows, chunkshape, byteorder, createparents, obj)
1065 description=description, title=title,
1066 filters=filters, expectedrows=expectedrows,
-> 1067 chunkshape=chunkshape, byteorder=byteorder)
1068
1069 if obj is not None:
/home/tony/.virtualenvs/dev2/lib/python2.7/site-packages/tables/table.pyc in __init__(self, parentnode, name, description, title, filters, expectedrows, chunkshape, byteorder, _log)
805 if new and isinstance(description, dict):
806 # Dictionary case
--> 807 self.description = Description(description)
808 elif new and (type(description) == type(IsDescription)
809 and issubclass(description, IsDescription)):
/home/tony/.virtualenvs/dev2/lib/python2.7/site-packages/tables/description.pyc in __init__(self, classdict, nestedlvl, validate)
493 'got: "%s". Please make use of the Col(), or '
494 'descendant, constructor to properly '
--> 495 'initialize columns.' % object)
496 object._v_pos = pos # Set the position of this object
497 object._v_parent = self # The parent description
TypeError: Passing an incorrect value to a table column. Expected a Col (or subclass) instance and got: "ObjectAtom()". Please make use of the Col(), or descendant, constructor to properly initialize columns.
My system info:
INSTALLED VERSIONS
------------------
commit: None
python: 2.7.6.final.0
python-bits: 64
OS: Linux
OS-release: 2.6.32-431.17.1.el6.x86_64
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8
pandas: 0.15.0
nose: 1.3.4
Cython: 0.20.1
numpy: 1.9.0
scipy: 0.14.0
statsmodels: 0.5.0
IPython: 2.3.0
sphinx: 1.2.2
patsy: 0.2.1
dateutil: 2.2
pytz: 2014.7
bottleneck: 0.8.0
tables: 3.1.1
numexpr: 2.4
matplotlib: 1.4.0
openpyxl: 1.8.6
xlrd: None
xlwt: None
xlsxwriter: None
lxml: 3.3.5
bs4: 4.3.2
html5lib: 0.999
httplib2: None
apiclient: None
rpy2: None
sqlalchemy: 0.9.4
pymysql: None
psycopg2: None
Comment From: jreback
ok, will try to get to this, but if you would like to dig in would be great.
This is actually a tiny bit tricky, because the index needs to be stringified and NOT stored as an object dtype (which is not allowed in a column in a table format; and in 'fixed' format its inefficent (and show the PerfWarning) because its pickled!))
so this needs to be a separate case
Comment From: jreback
closing as Panel deprecated