change DB structure

move to new DB layout with versioning
support bson format for category files

11 files changed, 707 insertions, 302 deletions

diff --git a/g_sorcery/bson/__init__.py b/g_sorcery/bson/__init__.py
new file mode 100644
index 0000000..4265cc3
--- /dev/null
+++ b/g_sorcery/bson/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/g_sorcery/bson/bson.py b/g_sorcery/bson/bson.py
new file mode 100644
index 0000000..fdb8bb9
--- /dev/null
+++ b/g_sorcery/bson/bson.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    bson.py
+    ~~~~~~~
+
+    bson file format support
+
+    :copyright: (c) 2015 by Jauhien Piatlicki
+    :license: GPL-2, see LICENSE for more details.
+"""
+
+import bson
+
+from g_sorcery.exceptions import FileJSONError
+from g_sorcery.fileutils import FileJSONData
+from g_sorcery.serialization import from_raw_serializable, to_raw_serializable
+
+class FileBSON(FileJSONData):
+    """
+    Class for BSON files. Supports custom JSON serialization
+    provided by g_sorcery.serialization.
+    """
+    def read_content(self):
+        """
+        Read BSON file.
+        """
+        content = {}
+        bcnt = None
+        with open(self.path, 'rb') as f:
+            bcnt = f.read()
+        if not bcnt:
+            raise FileJSONError('failed to read: ', self.path)
+        rawcnt = bson.BSON.decode(bcnt)
+        content = from_raw_serializable(rawcnt)
+        return content
+
+
+    def write_content(self, content):
+        """
+        Write BSON file.
+        """
+        rawcnt = to_raw_serializable(content)
+        bcnt = bson.BSON.encode(rawcnt)
+        with open(self.path, 'wb') as f:
+            f.write(bcnt)
diff --git a/g_sorcery/db_layout.py b/g_sorcery/db_layout.py
new file mode 100644
index 0000000..fe6f281
--- /dev/null
+++ b/g_sorcery/db_layout.py
@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    db_layout.py
+    ~~~~~~~~~~~~
+
+    package database file layout
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
+    :license: GPL-2, see LICENSE for more details.
+"""
+
+import hashlib
+import os
+import shutil
+
+from .exceptions import DBLayoutError, DBStructureError, FileJSONError, IntegrityError
+from .fileutils import FileJSON, hash_file
+
+CATEGORIES_FILE_NAME = 'categories'
+MANIFEST_FILE_NAME = 'manifest'
+METADATA_FILE_NAME = 'metadata'
+PACKAGES_FILE_NAME = 'packages'
+
+JSON_FILE_SUFFIX = 'json'
+BSON_FILE_SUFFIX = 'bson'
+
+class CategoryJSON(FileJSON):
+    """
+    Category file in JSON format.
+    """
+    def __init__(self, directory, category):
+        super(CategoryJSON, self).__init__(os.path.join(os.path.abspath(directory), category),
+                                           file_name(PACKAGES_FILE_NAME, JSON_FILE_SUFFIX))
+
+
+SUPPORTED_FILE_FORMATS = {JSON_FILE_SUFFIX: CategoryJSON}
+
+
+# bson module is optional, we should check if it is installed
+try:
+    from g_sorcery.bson.bson import FileBSON
+
+    class CategoryBSON(FileBSON):
+        """
+        Category file in BSON format.
+        """
+        def __init__(self, directory, category):
+            super(CategoryBSON, self).__init__(os.path.join(os.path.abspath(directory), category),
+                                               file_name(PACKAGES_FILE_NAME, BSON_FILE_SUFFIX))
+
+    SUPPORTED_FILE_FORMATS[BSON_FILE_SUFFIX] = CategoryBSON
+
+except ImportError as e:
+    pass
+
+
+def file_name(name, suffix=JSON_FILE_SUFFIX):
+    """
+    Return file name based on name and suffix.
+    """
+    return name + '.' + suffix
+
+
+class Manifest(FileJSON):
+    """
+    Manifest file.
+    """
+
+    def __init__(self, directory):
+        super(Manifest, self).__init__(os.path.abspath(directory), file_name(MANIFEST_FILE_NAME))
+
+    def check(self):
+        """
+        Check manifest.
+        """
+        manifest = self.read()
+
+        result = True
+        errors = []
+
+        names = [file_name(CATEGORIES_FILE_NAME)]
+        for name in names:
+            if not name in manifest:
+                raise DBLayoutError('Bad manifest: no ' + name + ' entry')
+
+        for name, value in manifest.items():
+            if hash_file(os.path.join(self.directory, name), hashlib.md5()) != \
+                value:
+                errors.append(name)
+
+        if errors:
+            result = False
+
+        return (result, errors)
+
+    def digest(self, mandatory_files):
+        """
+        Generate manifest.
+        """
+        if not file_name(CATEGORIES_FILE_NAME) in mandatory_files:
+            raise DBLayoutError('Categories file: ' + file_name(CATEGORIES_FILE_NAME) \
+                                + ' is not in the list of mandatory files')
+
+        categories = Categories(self.directory)
+        categories = categories.read()
+
+        manifest = {}
+
+        for name in mandatory_files:
+            manifest[name] = hash_file(os.path.join(self.directory, name),
+                                       hashlib.md5())
+
+        for category in categories:
+            category_path = os.path.join(self.directory, category)
+            if not os.path.isdir(category_path):
+                raise DBStructureError('Empty category: ' + category)
+            for root, _, files in os.walk(category_path):
+                for f in files:
+                    manifest[os.path.join(root[len(self.directory)+1:], f)] = \
+                    hash_file(os.path.join(root, f), hashlib.md5())
+
+        self.write(manifest)
+
+
+class Metadata(FileJSON):
+    """
+    Metadata file.
+    """
+    def __init__(self, directory):
+        super(Metadata, self).__init__(os.path.abspath(directory),
+                                       file_name(METADATA_FILE_NAME),
+                                       ['db_version', 'layout_version', 'category_format'])
+
+    def read(self):
+        """
+        Read metadata file.
+
+        If file doesn't exist, we have a legacy DB
+        with DB layout v. 0. Fill metadata appropriately.
+        """
+        if not os.path.exists(self.directory):
+            os.makedirs(self.directory)
+        content = {}
+        if not os.path.isfile(self.path):
+            content = {'db_version': 0, 'layout_version': 0, 'category_format': JSON_FILE_SUFFIX}
+        else:
+            content = self.read_content()
+            for key in self.mandatories:
+                if not key in content:
+                    raise FileJSONError('lack of mandatory key: ' + key)
+
+        return content
+
+
+class Categories(FileJSON):
+    """
+    Categories file.
+    """
+    def __init__(self, directory):
+        super(Categories, self).__init__(os.path.abspath(directory),
+                                         file_name(CATEGORIES_FILE_NAME))
+
+
+def get_layout(metadata):
+    """
+    Get layout parameters based on metadata.
+    """
+    layout_version = metadata['layout_version']
+    if layout_version == 0:
+        return (CategoryJSON, [file_name(CATEGORIES_FILE_NAME)])
+    elif layout_version == 1:
+        category_format = metadata['category_format']
+        try:
+            category_cls = SUPPORTED_FILE_FORMATS[category_format]
+        except KeyError:
+            raise DBLayoutError("unsupported packages file format: " + category_format)
+        return (category_cls, [file_name(CATEGORIES_FILE_NAME), file_name(METADATA_FILE_NAME)])
+    else:
+        raise DBLayoutError("unsupported DB layout version: " + layout_version)
+
+
+class DBLayout(object):
+    """
+    Filesystem DB layout.
+
+    Directory layout.
+    ~~~~~~~~~~~~~~~~~
+
+    For legacy DB layout v. 0:
+
+    db dir
+        manifest.json: database manifest
+        categories.json: information about categories
+        category1
+            packages.json: information about available packages
+        category2
+        ...
+
+    For DB layout v. 1:
+
+    db dir
+        manifest.json: database manifest
+        categories.json: information about categories
+        metadata.json: DB metadata
+        category1
+            packages.[b|j]son: information about available packages
+        category2
+        ...
+
+    Packages file can be in json or bson formats.
+    """
+
+    def __init__(self, directory):
+        self.directory = os.path.abspath(directory)
+        self.manifest = Manifest(self.directory)
+
+    def check_manifest(self):
+        """
+        Check manifest.
+        """
+        sane, errors = self.manifest.check()
+        if not sane:
+            raise IntegrityError('Manifest error: ' + str(errors))
+
+    def clean(self):
+        """
+        Remove DB files.
+        """
+        if os.path.exists(self.directory):
+            shutil.rmtree(self.directory)
+
+    def read(self):
+        """
+        Read DB files.
+
+        Returns a tuple with metadata, list of categories
+        and categories dictionary.
+        """
+        self.check_manifest()
+
+        metadata_f = Metadata(self.directory)
+        metadata = metadata_f.read()
+
+        category_cls, _ = get_layout(metadata)
+
+        categories_f = Categories(self.directory)
+        categories = categories_f.read()
+
+        packages = {}
+        for category in categories:
+            category_path = os.path.join(self.directory, category)
+            if not os.path.isdir(category_path):
+                raise DBLayoutError('Empty category: ' + category)
+            category_f = category_cls(self.directory, category)
+            pkgs = category_f.read()
+            if not pkgs:
+                raise DBLayoutError('Empty category: ' + category)
+            packages[category] = pkgs
+
+        return (metadata, categories, packages)
+
+    def write(self, metadata, categories, packages):
+        """
+        Write DB files.
+        """
+        category_cls, mandatory_files = get_layout(metadata)
+
+        self.clean()
+
+        if file_name(METADATA_FILE_NAME) in mandatory_files:
+            metadata_f = Metadata(self.directory)
+            metadata_f.write(metadata)
+
+        categories_f = Categories(self.directory)
+        categories_f.write(categories)
+
+        for category in categories:
+            category_f = category_cls(self.directory, category)
+            category_f.write(packages[category])
+
+        self.manifest.digest(mandatory_files)
diff --git a/g_sorcery/exceptions.py b/g_sorcery/exceptions.py
index a8d7238..4691ce6 100644
--- a/g_sorcery/exceptions.py
+++ b/g_sorcery/exceptions.py
@@ -4,10 +4,10 @@
 """
     exceptions.py
     ~~~~~~~~~~~~~
-    
+
     Exceptions hierarchy
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
@@ -17,6 +17,9 @@ class GSorceryError(Exception):
 class DBError(GSorceryError):
     pass
 
+class DBLayoutError(GSorceryError):
+    pass
+
 class InvalidKeyError(DBError):
     pass
 
@@ -49,3 +52,6 @@ class DigestError(GSorceryError):
 
 class DownloadingError(GSorceryError):
     pass
+
+class SerializationError(GSorceryError):
+    pass
diff --git a/g_sorcery/fileutils.py b/g_sorcery/fileutils.py
index 443206c..d783c8a 100644
--- a/g_sorcery/fileutils.py
+++ b/g_sorcery/fileutils.py
@@ -4,10 +4,10 @@
 """
     fileutils.py
     ~~~~~~~~~~~~
-    
+
     file utilities
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
@@ -15,17 +15,15 @@ import glob
 import json
 import hashlib
 import os
-import shutil
 import tarfile
 
 from .compatibility import TemporaryDirectory
 from .exceptions import FileJSONError, DownloadingError
 from .serialization import JSONSerializer, deserializeHook
 
-class FileJSON(object):
+class FileJSONData(object):
     """
-    Class for JSON files. Supports custom JSON serialization
-    provided by g_sorcery.serialization.
+    Class for files with JSON compatible data.
     """
     def __init__(self, directory, name, mandatories=None):
         """
@@ -33,7 +31,6 @@ class FileJSON(object):
             directory: File directory.
             name: File name.
             mandatories: List of requiered keys.
-            loadconv: Type change values on loading.
         """
         self.directory = os.path.abspath(directory)
         self.name = name
@@ -45,7 +42,7 @@ class FileJSON(object):
 
     def read(self):
         """
-        Read JSON file.
+        Read file.
         """
         if not os.path.exists(self.directory):
             os.makedirs(self.directory)
@@ -53,27 +50,58 @@ class FileJSON(object):
         if not os.path.isfile(self.path):
             for key in self.mandatories:
                 content[key] = ""
-            with open(self.path, 'w') as f:
-                json.dump(content, f, indent=2,
-                    sort_keys=True, cls=JSONSerializer)
+            self.write_content(content)
         else:
-            with open(self.path, 'r') as f:
-                content = json.load(f, object_hook=deserializeHook)
+            content = self.read_content()
             for key in self.mandatories:
                 if not key in content:
                     raise FileJSONError('lack of mandatory key: ' + key)
-        
+
         return content
 
+    def read_content(self):
+        """
+        Real read operation with deserialization. Should be overridden.
+        """
+        return []
+
     def write(self, content):
         """
-        Write JSON file.
+        Write file.
         """
         for key in self.mandatories:
             if not key in content:
                 raise FileJSONError('lack of mandatory key: ' + key)
         if not os.path.exists(self.directory):
             os.makedirs(self.directory)
+        self.write_content(content)
+
+    def write_content(self, content):
+        """
+        Real write operation with serialization. Should be overridden.
+        """
+        pass
+
+
+class FileJSON(FileJSONData):
+    """
+    Class for JSON files. Supports custom JSON serialization
+    provided by g_sorcery.serialization.
+    """
+
+    def read_content(self):
+        """
+        Read JSON file.
+        """
+        content = {}
+        with open(self.path, 'r') as f:
+            content = json.load(f, object_hook=deserializeHook)
+        return content
+
+    def write_content(self, content):
+        """
+        Write JSON file.
+        """
         with open(self.path, 'w') as f:
             json.dump(content, f, indent=2, sort_keys=True, cls=JSONSerializer)
 
@@ -149,7 +177,7 @@ class ManifestEntry(object):
 
     __slots__ = ('directory', 'name', 'ftype',
                  'size', 'sha256', 'sha512', 'whirlpool')
-    
+
     def __init__(self, directory, name, ftype):
         self.directory = directory
         self.name = name
diff --git a/g_sorcery/package_db.py b/g_sorcery/package_db.py
index f19f9d4..5eeeb63 100644
--- a/g_sorcery/package_db.py
+++ b/g_sorcery/package_db.py
@@ -4,45 +4,46 @@
 """
     package_db.py
     ~~~~~~~~~~~~~
-    
+
     package database
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
 import glob
-import hashlib
 import os
-import shutil
-import sys
 
 import portage
 
 from .compatibility import basestring, py2k, TemporaryDirectory
 
-from .exceptions import DBStructureError, IntegrityError, \
-     InvalidKeyError, SyncError
-from .fileutils import FileJSON, hash_file, load_remote_file, copy_all, wget
+from .db_layout import DBLayout, JSON_FILE_SUFFIX
+from .exceptions import DBError, DBStructureError, InvalidKeyError, SyncError
+from .fileutils import FileJSON, load_remote_file, copy_all, wget
 from .g_collections import Package
-from .logger import Logger, ProgressBar
+from .logger import Logger
 
 
 class PackageDB(object):
     """
     Package database.
-    Database is a directory and related data structure.
-
-    Directory layout.
-    ~~~~~~~~~~~~~~~~~
-    db dir
-        manifest.json: database manifest
-        categories.json: information about categories
-        category1
-            packages.json: information about available packages
-        category2
-        ...
-    
+    It uses DBLayout class to manipulate files that
+    contain DB stored on disk.
+
+    There are two versions of DB layout now:
+        0 -- legacy version
+        1 -- new version that supports DB structure versioning
+
+    DB structure itself has two versions:
+        0 -- legacy version, categories contain dictionary package name: versions dict
+        1 -- actual version, corresponds to the DB in memory:
+            DB is a dictionary with categories as keys.
+            Each category contains a dictionary with two entries:
+                common_data -- fields common to all the packages
+                packages -- dictionary with packages (content of category dictionary in v. 0)
+
+    For DB layout v. 0 only DB structure v. 0 is possible.
     """
 
     class Iterator(object):
@@ -50,68 +51,113 @@ class PackageDB(object):
         Iterator class over the package database.
         """
         def __init__(self, package_db):
-            self.pkg_iter = iter(package_db.database.items())
+            self.cats_iter = iter(package_db.database.items())
             try:
-                self.pkgname, self.vers_dict = next(self.pkg_iter)
+                self.cat_name, self.cat_data = next(self.cats_iter)
             except StopIteration:
-                self.pkgname, self.vers_dict = None, None
-            if self.vers_dict:
-                self.vers_iter = iter(self.vers_dict.items())
-            else:
-                self.vers_iter = None
+                self.set_to_end()
+                return
+
+            if not self.cat_data:
+                self.set_to_end()
+                return
+
+            self.pkgs_iter = iter(self.cat_data['packages'].items())
+            try:
+                self.pkg_name, self.pkg_data = next(self.pkgs_iter)
+            except StopIteration:
+                self.set_to_end()
+                return
+
+            if not self.pkg_data:
+                self.set_to_end()
+                return
+
+            self.vers_iter = iter(self.pkg_data.items())
+
+        def set_to_end(self):
+            self.cat_name, self.cat_data = None, None
+            self.pkgs_iter = None
+            self.pkg_name, self.pkg_data = None, None
+            self.vers_iter = None
 
         def __iter__(self):
             return self
 
         if py2k:
             def next(self):
-                if not self.vers_iter:
+                if not self.vers_iter or not self.pkgs_iter:
                     raise StopIteration
+
                 ver, ebuild_data = None, None
                 while not ver:
                     try:
                         ver, ebuild_data = next(self.vers_iter)
                     except StopIteration:
                         ver, ebuild_data = None, None
-
                     if not ver:
-                        self.pkgname, self.vers_dict = next(self.pkg_iter)
-                        self.vers_iter = iter(self.vers_dict.items())
+                        try:
+                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
+                            self.vers_iter = iter(self.pkg_data.items())
+                        except StopIteration:
+                            self.cat_name, self.cat_data = next(self.cats_iter)
+                            self.pkgs_iter = iter(self.cat_data['packages'].items())
+                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
+                            self.vers_iter = iter(self.pkg_data.items())
+
+                ebuild_data.update(self.cat_data['common_data'])
+                return (Package(self.cat_name, self.pkg_name, ver), ebuild_data)
 
-                category, name = self.pkgname.split('/')
-                return (Package(category, name, ver), ebuild_data)
         else:
             def __next__(self):
-                if not self.vers_iter:
+                if not self.vers_iter or not self.pkgs_iter:
                     raise StopIteration
+
                 ver, ebuild_data = None, None
                 while not ver:
                     try:
                         ver, ebuild_data = next(self.vers_iter)
                     except StopIteration:
                         ver, ebuild_data = None, None
-
                     if not ver:
-                        self.pkgname, self.vers_dict = next(self.pkg_iter)
-                        self.vers_iter = iter(self.vers_dict.items())
+                        try:
+                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
+                            self.vers_iter = iter(self.pkg_data.items())
+                        except StopIteration:
+                            self.cat_name, self.cat_data = next(self.cats_iter)
+                            self.pkgs_iter = iter(self.cat_data['packages'].items())
+                            self.pkg_name, self.pkg_data = next(self.pkgs_iter)
+                            self.vers_iter = iter(self.pkg_data.items())
 
-                category, name = self.pkgname.split('/')
-                return (Package(category, name, ver), ebuild_data)
+                ebuild_data.update(self.cat_data['common_data'])
+                return (Package(self.cat_name, self.pkg_name, ver), ebuild_data)
 
 
-    def __init__(self, directory):
-        """
-        Args:
-            directory: database directory.
-        """
+    def __init__(self, directory,
+                 preferred_layout_version=1,
+                 preferred_db_version=1,
+                 preferred_category_format=JSON_FILE_SUFFIX):
+
+        if preferred_layout_version == 0 \
+           and preferred_db_version != 0:
+            raise DBStructureError("Wrong DB version: " + preferred_db_version + \
+                                   ", with DB layout version 0 it can be only 0")
+
+        if not preferred_db_version in [0, 1]:
+            raise DBStructureError("Unsupported DB version: " + preferred_db_version)
+
         self.logger = Logger()
-        self.CATEGORIES_NAME = 'categories.json'
-        self.PACKAGES_NAME = 'packages.json'
         self.directory = os.path.abspath(directory)
+        self.preferred_layout_version = preferred_layout_version
+        self.preferred_db_version = preferred_db_version
+        self.preferred_category_format = preferred_category_format
+        self.db_layout = DBLayout(self.directory)
         self.reset_db()
 
+
     def __iter__(self):
-        return(PackageDB.Iterator(self))
+        return PackageDB.Iterator(self)
+
 
     def reset_db(self):
         """
@@ -120,6 +166,7 @@ class PackageDB(object):
         self.database = {}
         self.categories = {}
 
+
     def sync(self, db_uri):
         """
         Synchronize local database with remote database.
@@ -131,7 +178,7 @@ class PackageDB(object):
         download_dir = TemporaryDirectory()
         if wget(real_db_uri, download_dir.name):
             raise SyncError('sync failed: ' + real_db_uri)
-        
+
         temp_dir = TemporaryDirectory()
         for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')):
             self.logger.info("unpacking " + f_name)
@@ -140,19 +187,18 @@ class PackageDB(object):
         tempdb_dir = os.path.join(temp_dir.name, os.listdir(temp_dir.name)[0])
         tempdb = PackageDB(tempdb_dir)
 
-        if not tempdb.check_manifest()[0]:
-            raise IntegrityError('Manifest check failed.')
+        tempdb.db_layout.check_manifest()
 
         self.logger.info("copy files to an actual database")
         self.clean()
         copy_all(tempdb_dir, self.directory)
-        
-        if not self.check_manifest()[0]:
-            raise IntegrityError('Manifest check failed, db inconsistent.')
-                
+
+        self.db_layout.check_manifest()
+
         del download_dir
         del temp_dir
 
+
     def get_real_db_uri(self, db_uri):
         """
         Convert self.db_uri to URI where remote database can be
@@ -162,196 +208,61 @@ class PackageDB(object):
             URI of remote database file.
         """
         return db_uri
-            
-    def manifest(self):
-        """
-        Generate database manifest.
-        """
-        categories = FileJSON(self.directory, self.CATEGORIES_NAME, [])
-        categories = categories.read()
-        manifest = {}
-        names = [self.CATEGORIES_NAME]
-        for name in names:
-            manifest[name] = hash_file(os.path.join(self.directory, name),
-                                      hashlib.md5())
-        for category in categories:
-            category_path = os.path.join(self.directory, category)
-            if not os.path.isdir(category_path):
-                raise DBStructureError('Empty category: ' + category)
-            for root, dirs, files in os.walk(category_path):
-                for f in files:
-                    manifest[os.path.join(root[len(self.directory)+1:], f)] = \
-                    hash_file(os.path.join(root, f), hashlib.md5())
-        m_f = FileJSON(self.directory, 'manifest.json', [])
-        m_f.write(manifest)
-
-    def check_manifest(self):
-        """
-        Check database manifest.
 
-        Returns:
-            Tuple with first element containing result of manifest check
-            as boolean and second element containing list of files with errors.
-        """
-        self.logger.info("checking manifest")
-        m_f = FileJSON(self.directory, 'manifest.json', [])
-        manifest = m_f.read()
-
-        result = True
-        errors = []
-
-        names = [self.CATEGORIES_NAME]
-        for name in names:
-            if not name in manifest:
-                raise DBStructureError('Bad manifest: no ' + name + ' entry')
-
-        for name, value in manifest.items():
-            if hash_file(os.path.join(self.directory, name), hashlib.md5()) != \
-                value:
-                errors.append(name)
-
-        if errors:
-            result = False
-
-        return (result, errors)
 
     def clean(self):
         """
         Clean database.
         """
-        if os.path.exists(self.directory):
-            shutil.rmtree(self.directory)
+        self.db_layout.clean()
         self.reset_db()
-        self.write_and_manifest()
-
-    def write_and_manifest(self):
-        """
-        Write and digest database.
-        """
         self.write()
-        self.manifest()
+
 
     def write(self):
         """
-        Write database.
+        Write and digest database.
         """
-        categories_f = FileJSON(self.directory, self.CATEGORIES_NAME, [])
-        categories_f.write(self.categories)
-
         if self.database:
-            self.logger.info("writing database")
+            self.logger.info("writing database...")
+
+        metadata = {'db_version': self.preferred_db_version,
+                    'layout_version': self.preferred_layout_version,
+                    'category_format': self.preferred_category_format}
+
+        if self.preferred_db_version == 0:
+            packages = dict(self.database)
+            for category, cat_data in packages.items():
+                for _, versions in cat_data['packages'].items():
+                    for version, ebuild_data in versions.items():
+                        ebuild_data.update(cat_data['common_data'])
+                packages[category] = cat_data['packages']
+        else:
+            packages = dict(self.database)
 
-        progress_bar = ProgressBar(20, len(list(self.database)))
-        if self.database:
-            progress_bar.begin()
-
-        categories_content = {}
-        for category in self.categories:
-            categories_content[category] = {}
-        
-        for pkgname, versions in self.database.items():
-            category, name = pkgname.split('/')
-            if not category or (not category in self.categories):
-                raise DBStructureError('Non existent: ' + category)
-            categories_content[category][name] = {}
-            for version, content in versions.items():
-                categories_content[category][name][version] = content
-                self.additional_write_version(category, name, version)
-            self.additional_write_package(category, name)
-            progress_bar.increment()
-
-        for category in self.categories:
-            f = FileJSON(os.path.join(self.directory, category), self.PACKAGES_NAME, [])
-            f.write(categories_content[category])
-            self.additional_write_category(category)
-
-        self.additional_write()
+        self.db_layout.write(metadata, self.categories, packages)
 
         if self.database:
-            progress_bar.end()
-            print("")
-
-    def additional_write_version(self, category, package, version):
-        """
-        Hook to be overrided.
-        """
-        pass
-
-    def additional_write_package(self, category, package):
-        """
-        Hook to be overrided.
-        """
-        pass
-
-    def additional_write_category(self, category):
-        """
-        Hook to be overrided.
-        """
-        pass
+            self.logger.info("database written")
 
-    def additional_write(self):
-        """
-        Hook to be overrided.
-        """
-        pass
 
     def read(self):
         """
         Read database.
         """
-        sane, errors = self.check_manifest()
-        if not sane:
-            raise IntegrityError('Manifest error: ' + str(errors))
-        categories_f = FileJSON(self.directory, self.CATEGORIES_NAME, [])
-        self.categories = categories_f.read()
-        for category in self.categories:
-            category_path = os.path.join(self.directory, category)
-            if not os.path.isdir(category_path):
-                raise DBStructureError('Empty category: ' + category)
-            
-            f = FileJSON(category_path, self.PACKAGES_NAME, [])
-            packages = f.read()
-            if not packages:
-                raise DBStructureError('Empty category: ' + category)
-            
-            for name, versions in packages.items():
-                
-                if not versions:
-                    error_msg = 'Empty package: ' + category + '/' + name
-                    raise DBStructureError(error_msg)
-                
-                pkgname = category + '/' + name
-                self.database[pkgname] = versions
-                for version in versions:
-                    self.additional_read_version(category, name, version)
-                self.additional_read_package(category, name)
-            self.additional_read_category(category)
-        self.additional_read()
+        metadata, self.categories, packages = self.db_layout.read()
 
-    def additional_read_version(self, category, package, version):
-        """
-        Hook to be overrided.
-        """
-        pass
+        db_version = metadata['db_version']
+        self.database = packages
+        if db_version == 0:
+            for category, cat_data in self.database.items():
+                self.database[category] = {'common_data': {}, 'packages': cat_data}
+        elif db_version == 1:
+            pass
+        else:
+            raise DBStructureError("Unsupported DB version: " + db_version)
 
-    def additional_read_package(self, category, package):
-        """
-        Hook to be overrided.
-        """
-        pass
 
-    def additional_read_category(self, category):
-        """
-        Hook to be overrided.
-        """
-        pass
-
-    def additional_read(self):
-        """
-        Hook to be overrided.
-        """
-        pass
-        
     def add_category(self, category, description=None):
         """
         Add a category.
@@ -364,6 +275,7 @@ class PackageDB(object):
             description = {}
         self.categories[category] = description
 
+
     def add_package(self, package, ebuild_data=None):
         """
         Add a package.
@@ -374,15 +286,25 @@ class PackageDB(object):
         """
         if not ebuild_data:
             ebuild_data = {}
+
         category = package.category
         name = package.name
         version = package.version
-        pkgname = category + '/' + name
-        if category and not category in self.categories:
+
+        if not category or not name or not version:
+            raise DBError("wrong package: " + str(package))
+
+        if not category in self.categories:
             raise InvalidKeyError('Non-existent category: ' + category)
-        if pkgname and not pkgname in self.database:
-            self.database[pkgname] = {}
-        self.database[pkgname][version] = ebuild_data
+
+        if not category in self.database:
+            self.database[category] = {'common_data': {}, 'packages': {}}
+
+        if not name in self.database[category]['packages']:
+            self.database[category]['packages'][name] = {}
+
+        self.database[category]['packages'][name][version] = ebuild_data
+
 
     def list_categories(self):
         """
@@ -393,6 +315,7 @@ class PackageDB(object):
         """
         return list(self.categories)
 
+
     def in_category(self, category, name):
         """
         Tests whether a package is in a given category.
@@ -406,7 +329,12 @@ class PackageDB(object):
         """
         if not category or (not category in self.categories):
             raise InvalidKeyError('No such category: ' + category)
-        return (category + '/' + name) in self.database
+
+        if not category in self.database:
+            return False
+
+        return name in self.database[category]['packages']
+
 
     def list_package_names(self, category):
         """
@@ -420,9 +348,12 @@ class PackageDB(object):
         """
         if not category or (not category in self.categories):
             raise InvalidKeyError('No such category: ' + category)
-        res = [x.split('/')[1] for x in self.database
-               if x.split('/')[0] == category]
-        return res
+
+        if not category in self.database:
+            return []
+
+        return list(self.database[category]['packages'])
+
 
     def list_catpkg_names(self):
         """
@@ -431,7 +362,12 @@ class PackageDB(object):
         Returns:
             List with category/package entries.
         """
-        return list(self.database)
+        result = []
+        for category, cat_data in self.database.items():
+            for name in cat_data['packages']:
+                result.append(category + '/' + name)
+        return result
+
 
     def list_package_versions(self, category, name):
         """
@@ -446,10 +382,13 @@ class PackageDB(object):
         """
         if not category or (not category in self.categories):
             raise InvalidKeyError('No such category: ' + category)
-        pkgname = category + '/' + name
-        if not pkgname in self.database:
-            raise InvalidKeyError('No such package: ' + pkgname)
-        return list(self.database[pkgname])
+
+        if not category in self.database \
+           or not name in self.database[category]['packages']:
+            raise InvalidKeyError('No such package: ' + category + '/' + name)
+
+        return list(self.database[category]['packages'][name])
+
 
     def list_all_packages(self):
         """
@@ -459,12 +398,13 @@ class PackageDB(object):
             List of package_db.Package instances.
         """
         result = []
-        for pkgname, versions in self.database.items():
-            for version in versions:
-                category, name = pkgname.split('/')
-                result.append(Package(category, name, version))
+        for category, cat_data in self.database.items():
+            for name, versions in cat_data['packages'].items():
+                for version in versions:
+                    result.append(Package(category, name, version))
         return result
 
+
     def get_package_description(self, package):
         """
         Get package ebuild data.
@@ -476,8 +416,11 @@ class PackageDB(object):
             Dictionary with package ebuild data.
         """
         #a possible exception should be catched in the caller
-        return self.database[package.category \
-                             + '/' + package.name][package.version]
+        desc = dict(self.database[package.category]['packages']\
+                    [package.name][package.version])
+        desc.update(self.database[package.category]['common_data'])
+        return desc
+
 
     def get_max_version(self, category, name):
         """
@@ -490,10 +433,15 @@ class PackageDB(object):
         Returns:
             The recent version of a package.
         """
+        if not category or (not category in self.categories):
+            raise InvalidKeyError('No such category: ' + category)
+
+        if not category in self.database \
+           or not name in self.database[category]['packages']:
+            raise InvalidKeyError('No such package: ' + category + '/' + name)
+
         pkgname = category + '/' + name
-        if not pkgname in self.database:
-            raise InvalidKeyError('No such package: ' + pkgname)
-        versions = list(self.database[pkgname])
+        versions = list(self.database[category]['packages'][name])
         max_ver = versions[0]
         for version in versions[1:]:
             if portage.pkgcmp(portage.pkgsplit(pkgname + '-' + version),
@@ -510,8 +458,15 @@ class DBGenerator(object):
 
     __slots__ = ('package_db_class')
 
-    def __init__(self, package_db_class=PackageDB):
+    def __init__(self, package_db_class=PackageDB,
+                 preferred_layout_version=1,
+                 preferred_db_version=1,
+                 preferred_category_format=JSON_FILE_SUFFIX):
         self.package_db_class = package_db_class
+        self.preferred_layout_version = preferred_layout_version
+        self.preferred_db_version = preferred_db_version
+        self.preferred_category_format = preferred_category_format
+
 
     def __call__(self, directory, repository,
                  common_config=None, config=None, generate=True):
@@ -539,7 +494,10 @@ class DBGenerator(object):
             Package database.
         """
         db_path = os.path.join(directory, repository, "db")
-        pkg_db = self.package_db_class(db_path)
+        pkg_db = self.package_db_class(db_path,
+                                       preferred_layout_version=self.preferred_layout_version,
+                                       preferred_db_version=self.preferred_db_version,
+                                       preferred_category_format=self.preferred_category_format)
 
         config_f = FileJSON(os.path.join(directory, repository),
                             "config.json", [])
@@ -557,9 +515,10 @@ class DBGenerator(object):
         if generate:
             pkg_db.clean()
             self.generate_tree(pkg_db, common_config, config)
-            pkg_db.write_and_manifest()
+            pkg_db.write()
         return pkg_db
 
+
     def generate_tree(self, pkg_db, common_config, config):
         """
         Generate package entries.
@@ -739,7 +698,7 @@ class DBGenerator(object):
         Hook to convert external dependencies.
         """
         return dependency
-        
+
     def in_config(self, configs, list_name, value):
         """
         Check whether value is in config.
diff --git a/g_sorcery/serialization.py b/g_sorcery/serialization.py
index e051596..780de6f 100644
--- a/g_sorcery/serialization.py
+++ b/g_sorcery/serialization.py
@@ -4,38 +4,100 @@
 """
     serialization.py
     ~~~~~~~~~~~~~~~~
-    
+
     json serialization
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
 import json
 import importlib
 
+from .exceptions import SerializationError
+
+def step_to_raw_serializable(obj):
+    """
+    Make one step of convertion of object
+    to the type that is serializable
+    by the json library.
+
+    None return value signifies an error.
+    """
+    if hasattr(obj, "serialize"):
+        if hasattr(obj, "deserialize"):
+            module = obj.__class__.__module__
+            name = obj.__class__.__name__
+            value = obj.serialize()
+            return {"python_module" : module,
+                    "python_class" : name,
+                    "value" : value}
+        else:
+            return obj.serialize()
+    return None
+
+
+def to_raw_serializable(obj):
+    """
+    Convert object to the raw serializable type.
+    Logic is the same as in the standard json encoder.
+    """
+    if isinstance(obj, str) \
+       or obj is None \
+       or obj is True \
+       or obj is False \
+       or isinstance(obj, int) \
+       or isinstance(obj, float) \
+       or isinstance(obj, (list, tuple)) \
+       or isinstance(obj, dict):
+        return obj
+    else:
+        sobj = step_to_raw_serializable(obj)
+        if not sobj:
+            raise SerializationError('Non serializable object: ', sobj)
+        return to_raw_serializable(sobj)
+
+
+def step_from_raw_serializable(sobj):
+    """
+    Make one step of building of object from the
+    raw json serializable type.
+    """
+    if "python_class" in sobj:
+        module = importlib.import_module(sobj["python_module"])
+        cls = getattr(module, sobj["python_class"])
+        return cls.deserialize(sobj["value"])
+    return sobj
+
+
+def from_raw_serializable(sobj):
+    """
+    Build object from the raw serializable object.
+    """
+    if isinstance(sobj, dict):
+        res = {k: from_raw_serializable(v) for k, v in sobj.items()}
+        return step_from_raw_serializable(res)
+    elif isinstance(sobj, list):
+        return [from_raw_serializable(item) for item in sobj]
+    else:
+        return sobj
+
 
 class JSONSerializer(json.JSONEncoder):
     """
     Custom JSON encoder.
 
     Each serializable class should have a method serialize
-    that returns JSON serializable value. If class addfitionally
+    that returns JSON serializable value. If class additionally
     has a classmethod deserialize that it can be deserialized
     and additional metainformation is added to the resulting JSON.
     """
     def default(self, obj):
-        if hasattr(obj, "serialize"):
-            if hasattr(obj, "deserialize"):
-                module = obj.__class__.__module__
-                name = obj.__class__.__name__
-                value = obj.serialize()
-                return {"python_module" : module,
-                        "python_class" : name,
-                        "value" : value}
-            else:
-                return obj.serialize()
-        return json.JSONEncoder.default(self, obj)
+        res = step_to_raw_serializable(obj)
+        if res:
+            return res
+        else:
+            return json.JSONEncoder.default(self, obj)
 
 
 def deserializeHook(json_object):
@@ -46,8 +108,4 @@ def deserializeHook(json_object):
     that takes value (previously returned by serialize method) and transforms
     it into class instance.
     """
-    if "python_class" in json_object:
-        module = importlib.import_module(json_object["python_module"])
-        cls = getattr(module, json_object["python_class"])
-        return cls.deserialize(json_object["value"])
-    return json_object
+    return step_from_raw_serializable(json_object)
diff --git a/gs_db_tool/gs_db_tool.py b/gs_db_tool/gs_db_tool.py
index f949eae..d9087b8 100644
--- a/gs_db_tool/gs_db_tool.py
+++ b/gs_db_tool/gs_db_tool.py
@@ -4,10 +4,10 @@
 """
     gs_db_tool.py
     ~~~~~~~~~~~~~
-    
-    CLI to manipulate with package DB
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    CLI to manipulate package DB
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
@@ -61,7 +61,7 @@ def transform_db(function):
     def transformator(pkg_db, args):
         pkg_db.read()
         function(pkg_db, args)
-        pkg_db.write_and_manifest()
+        pkg_db.write()
     return transformator
 
 
diff --git a/scripts/all_pythons.sh b/scripts/all_pythons.sh
index 9160dd0..af4c1f1 100755
--- a/scripts/all_pythons.sh
+++ b/scripts/all_pythons.sh
@@ -2,7 +2,7 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-for VER in 2.7 3.2 3.3
+for VER in 2.7 3.3 3.4
 do
     echo
     echo "testing python${VER}"
diff --git a/setup.py b/setup.py
index 015ec76..866a38f 100644
--- a/setup.py
+++ b/setup.py
@@ -1,15 +1,38 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    setup.py
+    ~~~~~~~~
+
+    installation script
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
+    :license: GPL-2, see LICENSE for more details.
+"""
+
+import os
 
 from distutils.core import setup
 
+SELECTABLE = ['bson']
+
+use_defaults = ' '.join(list(SELECTABLE))
+USE = os.environ.get("USE", use_defaults).split()
+
+optional_modules = []
+for mod in SELECTABLE:
+    if mod in USE:
+        optional_modules.append('g_sorcery.%s' % mod)
+
 setup(name          = 'g-sorcery',
-      version       = '0.1',
+      version       = '0.2',
       description   = 'framework for automated ebuild generators',
       author        = 'Jauhien Piatlicki',
       author_email  = 'jauhien@gentoo.org',
-      packages      = ['g_sorcery', 'gs_db_tool'],
+      packages      = ['g_sorcery', 'gs_db_tool'] + optional_modules,
       package_data  = {'g_sorcery': ['data/*']},
       scripts       = ['bin/g-sorcery', 'bin/gs-db-tool'],
       data_files    = [('/etc/g-sorcery/', ['g-sorcery.cfg'])],
-      license       = 'GPL',
+      license       = 'GPL-2',
       )
diff --git a/tests/test_PackageDB.py b/tests/test_PackageDB.py
index 373fa4b..f73f006 100644
--- a/tests/test_PackageDB.py
+++ b/tests/test_PackageDB.py
@@ -4,10 +4,10 @@
 """
     test_PackageDB.py
     ~~~~~~~~~~~~~~~~
-    
+
     PackageDB test suite
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
@@ -44,7 +44,7 @@ class TestPackageDB(BaseTest):
         for package in packages:
             orig_db.add_package(package, ebuild_data)
 
-        orig_db.write_and_manifest()
+        orig_db.write()
         os.system("cd " + orig_tempdir.name + " && tar cvzf good.tar.gz db")
         os.system("echo invalid >> " + orig_tempdir.name + "/db/app-test1/packages.json")
         os.system("cd " + orig_tempdir.name + " && tar cvzf dummy.tar.gz db")
@@ -82,7 +82,7 @@ class TestPackageDB(BaseTest):
             self.assertEqual(data, ebuild_data)
             pkg_set.remove(package)
         self.assertTrue(not pkg_set)
-        
+
 
 def suite():
     suite = unittest.TestSuite()