Memory Dump

We saw in the Proxy version that some types of queries were not possible using the lazr.delegates package. We solved that problem using Storm’s Proxy objects but we lost easyness in the process.

This time we will try to combine both aproaches to get the best of both worlds.

The interface definitions do not change:

    >>> from zope.interface import Interface, Attribute
    >>> class IPerson(Interface):
    ...     name = Attribute("Name")
    ...
    >>> class ISecretAgent(IPerson):
    ...     passcode = Attribute("Passcode")
    ...
    >>> class ITeacher(IPerson):
    ...     school = Attribute("School")

Neither the Person class:

    >>> from zope.interface import implements
    >>> from zope.interface.verify import verifyClass
    >>> from lazr.delegates import delegates
    >>> from storm.locals import Store, Storm, Unicode, Int, Proxy, Reference
    >>> class Person(Storm):
    ...     implements(IPerson)
    ...
    ...     __storm_table__ = "person"
    ...
    ...     id = Int(allow_none=False, primary=True)
    ...     name = Unicode()
    ...     person_type = Int(allow_none=False)
    ...     _person = None
    ...
    ...     def __init__(self, store, name, person_class, **kwargs):
    ...         self.name = name
    ...         self.person_type = person_class.person_type
    ...         store.add(self)
    ...         self._person = person_class(self, **kwargs)
    ...
    ...     @property
    ...     def person(self):
    ...         if self._person is None:
    ...             assert self.id is not None
    ...             person_class = BasePerson.get_class(self.person_type)
    ...             self._person = Store.of(self).get(person_class, self.id)
    ...         return self._person
    ...
    >>> verifyClass(IPerson, Person)
    True

Now, the real magic is in the metaclass. We use it not only to register our subclasses (so the Person.person property can find them) but to automatically store the attributes we needed to set manually in our previous version: the person_id and person attributes and a proxy object for each Person attribute. It’s like we are reimplementing inheritance in Python but not really

    >>> from storm.properties import PropertyPublisherMeta
    >>> class PersonType(PropertyPublisherMeta):
    ...     def __init__(self, name, bases, dict):
    ...         if hasattr(self, '__storm_table__'):
    ...             # this need to be done before calling the superclass
    ...             # otherwise Storm will cry about not having a primary key
    ...             self.person_id = Int(allow_none=False, primary=True)
    ...
    ...         super(PersonType, self).__init__(name, bases, dict)
    ...
    ...         if not hasattr(self, '_person_types_registry'):
    ...             self._person_types_registry = {}
    ...         elif hasattr(self, '__storm_table__'):
    ...             key = len(self._person_types_registry)
    ...             self._person_types_registry[key] = self
    ...             self.person_type = key
    ...
    ...             self.person = Reference(self.person_id, Person.id)
    ...             self._add_proxy_properties()
    ...
    ...     def _add_proxy_properties(self):
    ...         for name in IPerson:
    ...             if not hasattr(self, name):
    ...                 remote_attr = getattr(Person, name)
    ...                 setattr(self, name, Proxy(self.person, remote_attr))
    ...
    ...     def get_class(self, person_type):
    ...         return self._person_types_registry[person_type]

Not our BasePerson is really simple

    >>> class BasePerson(Storm):
    ...     __metaclass__ = PersonType
    ...
    ...     def __init__(self, person):
    ...         self.person = person

And so are the subclasses. No repetition, so it is less prone to mistakes

    >>> class SecretAgent(BasePerson):
    ...     implements(ISecretAgent)
    ...
    ...     __storm_table__ = "secret_agent"
    ...     passcode = Unicode()
    ...
    ...     def __init__(self, person, passcode=None):
    ...         super(SecretAgent, self).__init__(person)
    ...         self.passcode = passcode
    ...
    >>> verifyClass(ISecretAgent, SecretAgent)
    True
    >>> class Teacher(BasePerson):
    ...     implements(ITeacher)
    ...
    ...     __storm_table__ = "teacher"
    ...     school = Unicode()
    ...
    ...     def __init__(self, person, school=None):
    ...         super(Teacher, self).__init__(person)
    ...         self.school = school
    ...
    >>> verifyClass(ITeacher, Teacher)
    True

Let’s make sure our queries work as expected:

    >>> from storm.locals import create_database
    >>> database = create_database("sqlite:")
    >>> store = Store(database)
    >>> result = store.execute("""
    ...     CREATE TABLE person (
    ...         id INTEGER PRIMARY KEY,
    ...         person_type INTEGER NOT NULL,
    ...         name TEXT NOT NULL)
    ... """)
    >>> result = store.execute("""
    ...     CREATE TABLE secret_agent (
    ...         person_id INTEGER PRIMARY KEY,
    ...         passcode TEXT)
    ... """)
    >>> result = store.execute("""
    ...     CREATE TABLE teacher (
    ...         person_id INTEGER PRIMARY KEY,
    ...         school TEXT)
    ... """)
    ...
    >>> secret_agent = Person(store, u"James Bond",
    ...                        SecretAgent, passcode=u"007")
    >>> ISecretAgent.providedBy(secret_agent.person)
    True
    >>> teacher = Person(store, u"Albus Dumbledore",
    ...                  Teacher, school=u"Hogwarts")
    >>> ITeacher.providedBy(teacher.person)
    True
    >>> store.commit()
    >>> del secret_agent
    >>> del teacher
    >>> store.rollback()
    >>> secret_agent = store.find(SecretAgent).one()
    >>> secret_agent.name, secret_agent.passcode
    (u'James Bond', u'007')
    >>> teacher = store.find(Teacher).one()
    >>> teacher.name, teacher.school
    (u'Albus Dumbledore', u'Hogwarts')
    >>> secret_agent = store.find(SecretAgent, SecretAgent.name==u'James Bond').one()
    >>> secret_agent.passcode
    u'007'
    >>> teacher = store.find(Teacher, Teacher.school==u'Hogwarts').one()
    >>> teacher.name
    u'Albus Dumbledore'

So we made it by using a metaclass that automatically generate a bunch of attributes.

One problem that we had in the first version of our inheritance by composition pattern was that we could not make storm queries using the subclasses. In other words, the following would return None::

    secret_agent = store.find(SecretAgent, SecretAgent.name==u'James Bond').one()

The reason is that the expresion SecretAgent.name would resolve to a Passthrough lazr.delegates object that Storm does not know how to handle.

This time we will try to fix this problem using a manually generated version of our classes using storm’s Proxy objects.

The interface definitions do not change:

    >>> from zope.interface import Interface, Attribute
    >>> class IPerson(Interface):
    ...     name = Attribute("Name")
    ...
    >>> class ISecretAgent(IPerson):
    ...     passcode = Attribute("Passcode")
    ...
    >>> class ITeacher(IPerson):
    ...     school = Attribute("School")

Neither the Person class:

    >>> from zope.interface import implements
    >>> from zope.interface.verify import verifyClass
    >>> from lazr.delegates import delegates
    >>> from storm.locals import Store, Storm, Unicode, Int, Proxy, Reference
    >>> class Person(Storm):
    ...     implements(IPerson)
    ...
    ...     __storm_table__ = "person"
    ...
    ...     id = Int(allow_none=False, primary=True)
    ...     name = Unicode()
    ...     person_type = Int(allow_none=False)
    ...     _person = None
    ...
    ...     def __init__(self, store, name, person_class, **kwargs):
    ...         self.name = name
    ...         self.person_type = person_class.person_type
    ...         store.add(self)
    ...         self._person = person_class(self, **kwargs)
    ...
    ...     @property
    ...     def person(self):
    ...         if self._person is None:
    ...             assert self.id is not None
    ...             person_class = BasePerson.get_class(self.person_type)
    ...             self._person = Store.of(self).get(person_class, self.id)
    ...         return self._person
    >>> verifyClass(IPerson, Person)
    True

Neither our custom metaclass:

    >>> from storm.properties import PropertyPublisherMeta
    >>> class PersonType(PropertyPublisherMeta):
    ...     def __init__(self, name, bases, dict):
    ...         super(PersonType, self).__init__(name, bases, dict)
    ...         if not hasattr(self, '_person_types_registry'):
    ...             self._person_types_registry = {}
    ...         elif hasattr(self, '__storm_table__'):
    ...             key = len(self._person_types_registry)
    ...             self._person_types_registry[key] = self
    ...             self.person_type = key
    ...
    ...     def get_class(self, person_type):
    ...         return self._person_types_registry[person_type]

Here start the changes. Our BasePerson class does not have the delegates call and does not define the person_id attribute and person reference.

    >>> class BasePerson(Storm):
    ...     __metaclass__ = PersonType
    ...
    ...     def __init__(self, person):
    ...         self.person = person

Instead we define the person_id and person reference in each subclass and also we define each attribute of the Person base class as a proxy to the related attribute of the person reference.

    >>> class SecretAgent(BasePerson):
    ...     implements(ISecretAgent)
    ...
    ...     __storm_table__ = "secret_agent"
    ...     passcode = Unicode()
    ...     person_id = Int(allow_none=False, primary=True)
    ...     person = Reference(person_id, Person.id)
    ...     name = Proxy(person, Person.name)
    ...
    ...     def __init__(self, person, passcode=None):
    ...         super(SecretAgent, self).__init__(person)
    ...         self.passcode = passcode
    >>> verifyClass(ISecretAgent, SecretAgent)
    True

We do it again for the Teacher class:

    >>> class Teacher(BasePerson):
    ...     implements(ITeacher)
    ...
    ...     __storm_table__ = "teacher"
    ...     school = Unicode()
    ...     person_id = Int(allow_none=False, primary=True)
    ...     person = Reference(person_id, Person.id)
    ...     name = Proxy(person, Person.name)
    ...
    ...     def __init__(self, person, school=None):
    ...         super(Teacher, self).__init__(person)
    ...         self.school = school
    >>> verifyClass(ITeacher, Teacher)
    True

Time to test the database storage:

    >>> from storm.locals import create_database
    >>> database = create_database("sqlite:")
    >>> store = Store(database)
    >>> result = store.execute("""
    ...     CREATE TABLE person (
    ...         id INTEGER PRIMARY KEY,
    ...         person_type INTEGER NOT NULL,
    ...         name TEXT NOT NULL)
    ... """)
    >>> result = store.execute("""
    ...     CREATE TABLE secret_agent (
    ...         person_id INTEGER PRIMARY KEY,
    ...         passcode TEXT)
    ... """)
    >>> result = store.execute("""
    ...     CREATE TABLE teacher (
    ...         person_id INTEGER PRIMARY KEY,
    ...         school TEXT)
    ... """)
    >>> secret_agent = Person(store, u"James Bond",
    ...                        SecretAgent, passcode=u"007")
    >>> ISecretAgent.providedBy(secret_agent.person)
    True
    >>> teacher = Person(store, u"Albus Dumbledore",
    ...                  Teacher, school=u"Hogwarts")
    >>> ITeacher.providedBy(teacher.person)
    True
    >>> store.commit()

And what’s more important, all this changes should make possible to do the query with the subclass:

    >>> del secret_agent
    >>> del teacher
    >>> store.rollback()
    >>> secret_agent = store.find(SecretAgent).one()
    >>> secret_agent.name, secret_agent.passcode
    (u'James Bond', u'007')
    >>> teacher = store.find(Teacher).one()
    >>> teacher.name, teacher.school
    (u'Albus Dumbledore', u'Hogwarts')
    >>> secret_agent = store.find(SecretAgent, SecretAgent.name==u'James Bond').one()
    >>> secret_agent.passcode
    u'007'

We have improved the power of the pattern but now it is much more verbose to write each subclass since we need to repeat a lot of things. Note that we can not move the definition of the person_id and person attributes to the BasePerson aux class because Storm will tell us that it lacks the __storm_table__ attribute. In other words: storm does not allow attributes in abstract classes.

When using a ORM one of the most common problems is how to map your beautiful
application class inheritance to the database. The specific scenario I’m going
to describe is when you have an abstract base class with some fields and you
want to store those fields in a database table. Then all of its subclasses will
have their own table and to get the data of an instance you will need to make
a join between the base table and the subclass table.

One pattern to accomplish this is the infoheritance pattern described in the
Storm documentation. I’ll show you here a slightly modified version of this
pattern that uses the package lazr.delegates to make the user code easier by
hiding the fact that we are using composition to model inheritance.

So let’s start with some interface definition showing our model hierarchy:

    >>> from zope.interface import Interface, Attribute
    >>> class IPerson(Interface):
    ...     name = Attribute("Name")
    ...
    >>> class ISecretAgent(IPerson):
    ...     passcode = Attribute("Passcode")
    ...
    >>> class ITeacher(IPerson):
    ...     school = Attribute("School")

No rocket science so far. Let’s write now the implementation of the
IPerson interface. This will be our abstract base class. Pay special
attention to the person property which dynamically retrieve the subclass
instance by using composition.

    >>> from zope.interface import implements
    >>> from zope.interface.verify import verifyClass
    >>> from lazr.delegates import delegates
    >>> from storm.locals import Store, Storm, Unicode, Int, Reference
    >>> class Person(Storm):
    ...     implements(IPerson)
    ...
    ...     __storm_table__ = "person"
    ...
    ...     id = Int(allow_none=False, primary=True)
    ...     name = Unicode()
    ...     person_type = Int(allow_none=False)
    ...     _person = None
    ...
    ...     def __init__(self, store, name, person_class, **kwargs):
    ...         self.name = name
    ...         self.person_type = person_class.person_type
    ...         store.add(self)
    ...         self._person = person_class(self, **kwargs)
    ...
    ...     @property
    ...     def person(self):
    ...         if self._person is None:
    ...             assert self.id is not None
    ...             person_class = BasePerson.get_class(self.person_type)
    ...             self._person = Store.of(self).get(person_class, self.id)
    ...         return self._person
    >>> verifyClass(IPerson, Person)
    True

We will also use a custom metaclass (based on Storm metaclass) so it will
automatically register our subclasses. This is necessary for the dynamic
person property of the Person class where we map integer (stored in the
database) to classes (stored in the source code).

    >>> from storm.properties import PropertyPublisherMeta
    >>> class PersonType(PropertyPublisherMeta):
    ...     def __init__(self, name, bases, dict):
    ...         super(PersonType, self).__init__(name, bases, dict)
    ...         if not hasattr(self, '_person_types_registry'):
    ...             self._person_types_registry = {}
    ...         elif hasattr(self, '__storm_table__'):
    ...             key = len(self._person_types_registry)
    ...             self._person_types_registry[key] = self
    ...             self.person_type = key
    ...
    ...     def get_class(self, person_type):
    ...         return self._person_types_registry[person_type]

Now we define a convenience base class for our subclasses. Remember that
the subclasses don’t really inherit from the Person class because we
are using composition. This is where the lazr.delegates.delegates function
comes to rescue. By saying that we delegate the implementation of the
IPerson interface to the ‘person’ attribute, the instances of these subclasses
will look like they really are subclasses of Person, not just from BasePerson.

    >>> class BasePerson(Storm):
    ...     __metaclass__ = PersonType
    ...     delegates(IPerson, "person")
    ...
    ...     person_id = Int(allow_none=False, primary=True)
    ...     person = Reference(person_id, "Person.id")
    ...
    ...     def __init__(self, person):
    ...         self.person = person
    ...
    >>> class SecretAgent(BasePerson):
    ...     implements(ISecretAgent)
    ...
    ...     __storm_table__ = "secret_agent"
    ...     passcode = Unicode()
    ...
    ...     def __init__(self, person, passcode=None):
    ...         super(SecretAgent, self).__init__(person)
    ...         self.passcode = passcode

Here is the magic: we have an interface hierarchy and a different
hierarchy for the classes implementing them. But it’s all transparent
and our classes implement the full hierarchy:

    >>> verifyClass(ISecretAgent, SecretAgent)
    True
    >>> class Teacher(BasePerson):
    ...     implements(ITeacher)
    ...
    ...     __storm_table__ = "teacher"
    ...     school = Unicode()
    ...
    ...     def __init__(self, person, school=None):
    ...         super(Teacher, self).__init__(person)
    ...         self.school = school
    ...
    >>> verifyClass(ITeacher, Teacher)
    True

Now let’s try storing the objects in the database. We will use a
sqlite in memory database for this example.

    >>> from storm.locals import create_database
    >>> database = create_database("sqlite:")
    >>> store = Store(database)
    >>> result = store.execute("""
    ...     CREATE TABLE person (
    ...         id INTEGER PRIMARY KEY,
    ...         person_type INTEGER NOT NULL,
    ...         name TEXT NOT NULL)
    ... """)
    >>> result = store.execute("""
    ...     CREATE TABLE secret_agent (
    ...         person_id INTEGER PRIMARY KEY,
    ...         passcode TEXT)
    ... """)
    >>> result = store.execute("""
    ...     CREATE TABLE teacher (
    ...         person_id INTEGER PRIMARY KEY,
    ...         school TEXT)
    ... """)
    >>> secret_agent = Person(store, u"James Bond",
    ...                        SecretAgent, passcode=u"007")
    >>> ISecretAgent.providedBy(secret_agent.person)
    True
    >>> teacher = Person(store, u"Albus Dumbledore",
    ...                  Teacher, school=u"Hogwarts")
    >>> ITeacher.providedBy(teacher.person)
    True
    >>> store.commit()

The objects are now saved in the database. Let’s destroy them and
retrieve them back using some queries:

    >>> del secret_agent
    >>> del teacher
    >>> store.rollback()
    >>> secret_agent = store.find(SecretAgent).one()
    >>> secret_agent.name, secret_agent.passcode
    (u'James Bond', u'007')
    >>> teacher = store.find(Teacher).one()
    >>> teacher.name, teacher.school
    (u'Albus Dumbledore', u'Hogwarts')

The great thing is that we can write teacher.name without having to
write teacher.person.name (which also works by the way) effectively
hiding the implementation detail of composition.

Just one final note: Bear in mind that you should not have very deep
hierarchies or you may get a lot of JOINs and a slow application.

You can download a couple of files to test this on your own computer. You just need storm 0.14 and lazr.delegates 1.0, both of which can be installed with easy_install:

• Doctests
• Test runner