From c1ae721a98f18066066f6545609cc08d62e00a33 Mon Sep 17 00:00:00 2001 From: Andreas Balogh Date: Mon, 3 Oct 2016 15:21:17 +0200 Subject: [PATCH] initial commit --- .project | 17 +++ .pydevproject | 9 ++ LICENSE | 21 +++ src/tk_viewer.py | 75 +++++++++++ src/wx_viewer.py | 330 +++++++++++++++++++++++++++++++++++++++++++++++ src/zcm.py | 97 ++++++++++++++ test/test_zcm.py | 109 ++++++++++++++++ 7 files changed, 658 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 LICENSE create mode 100644 src/tk_viewer.py create mode 100644 src/wx_viewer.py create mode 100644 src/zcm.py create mode 100644 test/test_zcm.py diff --git a/.project b/.project new file mode 100644 index 0000000..82e1a40 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + zodb_tools + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..49472eb --- /dev/null +++ b/.pydevproject @@ -0,0 +1,9 @@ + + + +/${PROJECT_DIR_NAME}/src +/${PROJECT_DIR_NAME}/test + +python 3.0 +Default + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b6d0719 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Andreas Balogh + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/tk_viewer.py b/src/tk_viewer.py new file mode 100644 index 0000000..f32055b --- /dev/null +++ b/src/tk_viewer.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3.5 +# encoding: utf8 + +# Copyright (c) 2016 Andreas +# See LICENSE for details. + +""" tk ZODB viewer ui """ + +from argparse import ArgumentParser +from configparser import ConfigParser +import logging +import os +from pprint import pprint +import socket +import sys + +from BTrees.OOBTree import OOBTree +from persistent.list import PersistentList as PList +from persistent.mapping import PersistentMapping as PDict +import transaction +from zcm import ZDatabase, ZConnection +from threading import Thread, Event + +import datetime as dt +from queue import Queue + +# from BTrees.IOBTree import IOBTree +# from BTrees.IOBTree import IOBTree +# from persistent.list import PersistentList as PList +LOG = logging.getLogger(__name__) + + +def gui(argv=None): + """ command line interface """ + if argv is None: + argv = sys.argv + # parse options and arguments + parser = ArgumentParser(description="tumblr post incremental photo ripper") + parser.add_argument("command", choices=["pack", "load_mdh"], help="command") + parser.add_argument("--config", default="../etc/config.ini", + help="machine configuration file [default: %(default)s]") + parser.add_argument( + "--zmd", default="../var/md.zodb", help="zodb market data [default: %(default)s]") + parser.add_argument("-v", "--verbose", action="store_true", help="debug output") + args = parser.parse_args(argv[1:]) + argd = vars(args) + # main program + LOG.info("%s %s", os.path.basename(argv[0]), " ".join(argv[1:])) + # process config file + config = ConfigParser() + try: + config.read(args.config) + except OSError: + LOG.warn("config file {} missing, using defaults".format(args.config)) + else: + hostname = socket.gethostname() + try: + argd.update(config[hostname]) + except KeyError: + LOG.warn("no section for {} in {}".format(hostname, args.config)) + # debug logging + if args.verbose: + LOG.setLevel(logging.DEBUG) + pprint(argd) + # prepare queues and threads + func = globals()[args.command] + func(args) + LOG.info("done.") + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, + format='%(asctime)s.%(msecs)03i [%(thread)i] %(levelname).4s %(funcName)10s: %(message)s', + datefmt='%H:%M:%S') + sys.exit(gui()) diff --git a/src/wx_viewer.py b/src/wx_viewer.py new file mode 100644 index 0000000..6fb4100 --- /dev/null +++ b/src/wx_viewer.py @@ -0,0 +1,330 @@ +# {{{ http://code.activestate.com/recipes/409012/ (r2) +"""wxView.py - a simple view for ZODB files + +TODO: + -Support ZEO + -Rewrite/extend to use the builtin HTTP server a la pydoc +""" +from BTrees.OOBTree import OOBTree +from ZODB import FileStorage, DB +from persistent.list import PersistentList as PList +from persistent.mapping import PersistentMapping as PDict +import UserDict +import collections +import locale +import os +import transaction +import wx + + +def close_zodb(DataBase): + """Closes the ZODB. + + This function MUST be called at the end of each program !!! + See open_zodb() for a description of the argument. + """ + transaction.abort() + DataBase[1].close() + DataBase[2].close() + DataBase[3].close() + return True + +def open_zodb(Path): + """Open ZODB. + + Returns a tuple consisting of:(root,connection,db,storage) + The same tuple must be passed to close_zodb() in order to close the DB. + """ + # Connect to DB + storage = FileStorage.FileStorage(Path) + db = DB(storage) + connection = db.open() + root = connection.root() + return (root,connection,db,storage) + +def save_pos(win, cfg): + """Save a window position to the registry""" + (xpos, ypos) = win.GetPositionTuple() + (width, height) = win.GetSizeTuple() + cfg.WriteInt('xpos', xpos) + cfg.WriteInt('ypos', ypos) + cfg.WriteInt('width', width) + cfg.WriteInt('height', height) + +def set_pos(win, cfg): + """Restore a window to a position from the registry""" + xpos = cfg.ReadInt('xpos', -1) + ypos = cfg.ReadInt('ypos', -1) + width = cfg.ReadInt('width', -1) + height = cfg.ReadInt('height', -1) + win.SetDimensions(xpos, ypos, width, height) + +class ZODBFrame(wx.Frame): + def __init__(self, *args, **kwds): + # begin wxGlade: ZODBFrame.__init__ + kwds["style"] = wx.DEFAULT_FRAME_STYLE + wx.Frame.__init__(self, *args, **kwds) + self.window_1 = wx.SplitterWindow(self, -1, + style=wx.SP_3D|wx.SP_BORDER) + self.panel_1 = wx.Panel(self.window_1, -1) + self.window_1_pane_1 = wx.Panel(self.window_1, -1) + + self.wxcfg = wx.Config() + + # Menu Bar + self.mb = wx.MenuBar() + self.SetMenuBar(self.mb) + self.mnuFile = wx.Menu() + self.mnuOpen = wx.MenuItem(self.mnuFile, wx.ID_OPEN, "&Open\tCtrl-O", + "", wx.ITEM_NORMAL) + self.mnuFile.AppendItem(self.mnuOpen) + self.mnuFile.Append(wx.ID_CLOSE, "&Close", "", wx.ITEM_NORMAL) + self.mnuFile.AppendSeparator() + self.mnuFile.Append(wx.ID_EXIT, "E&xit", "", wx.ITEM_NORMAL) + self.mb.Append(self.mnuFile, "&File") + # Menu Bar end + self.sb = self.CreateStatusBar(1, wx.ST_SIZEGRIP) + self.db_layout_tree = wx.TreeCtrl(self.window_1_pane_1, -1, + style=wx.TR_HAS_BUTTONS| + wx.TR_LINES_AT_ROOT| + wx.TR_DEFAULT_STYLE| + wx.SUNKEN_BORDER) + self.label_1 = wx.StaticText(self.panel_1, -1, "Data Type:") + self.txtType = wx.StaticText(self.panel_1, -1, "txtType") + self.txtData = wx.TextCtrl(self.panel_1, -1, "", style=wx.TE_MULTILINE) + + self.__set_properties() + self.__do_layout() + # end wxGlade + + self.__create_image_list() + self.__create_file_history() + self.__set_window_position() + self.__set_bindings() + + self.db = None + self.root = None + + def __create_file_history(self): + self.file_history = wx.FileHistory() + self.file_history.UseMenu(self.mnuFile) + old_path = self.wxcfg.GetPath() + self.wxcfg.SetPath('/RecentFiles') + self.file_history.Load(self.wxcfg) + self.wxcfg.SetPath(old_path) + self._need_save = False + + def __create_image_list(self): + """Setup our image list for the tree control""" + isz = (16, 16) + il = wx.ImageList(*isz) + self.folder_idx = il.Add(wx.ArtProvider_GetBitmap(wx.ART_FOLDER, + wx.ART_OTHER, isz)) + self.folder_open_idx = il.Add(wx.ArtProvider_GetBitmap(wx.ART_FILE_OPEN, + wx.ART_OTHER, isz)) + self.file_idx = il.Add(wx.ArtProvider_GetBitmap(wx.ART_REPORT_VIEW, + wx.ART_OTHER, isz)) + self.il = il + + def __set_bindings(self): + self.Bind(wx.EVT_CLOSE, self.onExit) + self.Bind(wx.EVT_MENU, self.onExit, id=wx.ID_EXIT) + self.Bind(wx.EVT_MENU, self.doOpen, id=wx.ID_OPEN) + self.Bind(wx.EVT_MENU, self.doClose, id=wx.ID_CLOSE) + self.Bind(wx.EVT_MENU_RANGE, self.doFileHistory, id=wx.ID_FILE1, + id2=wx.ID_FILE9) + + self.db_layout_tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.onSelChange) + + def __set_properties(self): + # begin wxGlade: ZODBFrame.__set_properties + self.SetTitle("ZODB Viewer") + self.sb.SetStatusWidths([-1]) + # statusbar fields + sb_fields = [""] + for i in range(len(sb_fields)): + self.sb.SetStatusText(sb_fields[i], i) + self.txtType.SetFont(wx.Font(12, wx.MODERN, wx.NORMAL, wx.BOLD, 0, + "Courier New")) + self.txtData.SetFont(wx.Font(12, wx.MODERN, wx.NORMAL, wx.NORMAL, 0, + "Courier New")) + # end wxGlade + + def __set_window_position(self): + self.wxcfg = wx.Config() + old_path = self.wxcfg.GetPath() + self.wxcfg.SetPath('/Window Information') + set_pos(self, self.wxcfg) + self.wxcfg.SetPath(old_path) + + def __do_layout(self): + # begin wxGlade: ZODBFrame.__do_layout + sizer_1 = wx.BoxSizer(wx.HORIZONTAL) + grid_sizer_2 = wx.FlexGridSizer(2, 1, 5, 0) + grid_sizer_3 = wx.FlexGridSizer(1, 2, 0, 5) + sizer_2 = wx.BoxSizer(wx.HORIZONTAL) + sizer_2.Add(self.db_layout_tree, 1, wx.EXPAND, 0) + self.window_1_pane_1.SetAutoLayout(True) + self.window_1_pane_1.SetSizer(sizer_2) + sizer_2.Fit(self.window_1_pane_1) + sizer_2.SetSizeHints(self.window_1_pane_1) + grid_sizer_3.Add(self.label_1, 0, wx.FIXED_MINSIZE, 0) + grid_sizer_3.Add(self.txtType, 0, wx.FIXED_MINSIZE, 0) + grid_sizer_3.AddGrowableCol(1) + grid_sizer_2.Add(grid_sizer_3, 1, wx.EXPAND, 0) + grid_sizer_2.Add(self.txtData, 0, wx.EXPAND|wx.FIXED_MINSIZE, 0) + self.panel_1.SetAutoLayout(True) + self.panel_1.SetSizer(grid_sizer_2) + grid_sizer_2.Fit(self.panel_1) + grid_sizer_2.SetSizeHints(self.panel_1) + grid_sizer_2.AddGrowableRow(1) + grid_sizer_2.AddGrowableCol(0) + self.window_1.SplitVertically(self.window_1_pane_1, self.panel_1) + sizer_1.Add(self.window_1, 1, wx.EXPAND, 0) + self.SetAutoLayout(True) + self.SetSizer(sizer_1) + sizer_1.Fit(self) + sizer_1.SetSizeHints(self) + self.Layout() + # end wxGlade + + def _set_child_icons(self, c, d): + """Set the appropriate icons for a given tree child + + c + The child we are updating + + d + The data associated with the child + """ + if isinstance(d, (dict, UserDict.UserDict, OOBTree)): + self.db_layout_tree.SetItemImage(c, + self.folder_idx, + wx.TreeItemIcon_Normal) + self.db_layout_tree.SetItemImage(c, + self.folder_open_idx, + wx.TreeItemIcon_Expanded) + else: + self.db_layout_tree.SetItemImage(c, self.file_idx, + wx.TreeItemIcon_Normal) + + def createTree(self, filename): + """Create a new tree structure for when we open a file""" + self.doClose() + + self.db = open_zodb(filename) + self.db_layout_tree.SetImageList(self.il) + + self.root = self.db_layout_tree.AddRoot(os.path.basename(filename)) + self.db_layout_tree.SetPyData(self.root, self.db[0]) + self.db_layout_tree.SetItemImage(self.root, self.folder_idx, + wx.TreeItemIcon_Normal) + self.db_layout_tree.SetItemImage(self.root, self.folder_open_idx, + wx.TreeItemIcon_Expanded) + + db = self.db[0] + for key in list(db.keys()): + child = self.db_layout_tree.AppendItem(self.root, key) + self.db_layout_tree.SetPyData(child, db[key]) + if isinstance(db[key], dict) or isinstance(db[key], list): + self.db_layout_tree.SetItemImage(child, self.folder_idx, + wx.TreeItemIcon_Normal) + self.db_layout_tree.SetItemImage(child, self.folder_open_idx, + wx.TreeItemIcon_Expanded) + else: + self.db_layout_tree.SetItemImage(child, self.file_idx, + wx.TreeItemIcon_Normal) + + self.db_layout_tree.Expand(self.root) + + def doClose(self, *event): + """Close the current file and clear the screen""" + if self.db: + close_zodb(self.db) + self.db = None + if self.root: + self.db_layout_tree.DeleteAllItems() + self.txtType.SetLabel('') + self.txtData.Clear() + + def doFileHistory(self, event): + """Open a file from file history""" + file_number = event.GetId() - wx.ID_FILE1 + filename = self.file_history.GetHistoryFile(file_number) + self.createTree(filename) + + def doOpen(self, *event): + """Open a file from the file system""" + # Select and open the ZODB file object. + dlg = wx.FileDialog(self, message="Choose a file", + defaultFile="", style=wx.OPEN | wx.CHANGE_DIR) + + if dlg.ShowModal() == wx.ID_OK: + # This returns a Python list of files that were selected. + filename = dlg.GetPath() + self.createTree(filename) + self.file_history.AddFileToHistory(filename) + dlg.Destroy() + + def onExit(self, event): + """Exit the program""" + self.doClose() + + old_path = self.wxcfg.GetPath() + self.wxcfg.SetPath('/RecentFiles') + self.file_history.Save(self.wxcfg) + self.wxcfg.SetPath(old_path) + + old_path = self.wxcfg.GetPath() + self.wxcfg.SetPath('/Window Information') + save_pos(self, self.wxcfg) + self.wxcfg.SetPath(old_path) + self.Destroy() + + def onSelChange(self, event): + """Select a new tree node, loading it if needed""" + item = event.GetItem() + data = self.db_layout_tree.GetPyData(item) + if isinstance(data, (dict, UserDict.UserDict, OOBTree)): + self.txtData.Clear() + self.txtType.SetLabel(str(type(data))) + if hasattr(data, 'wx_str'): + self.txtData.AppendText(data.wx_str()) + if not self.db_layout_tree.ItemHasChildren(item): + keys = list(data.keys()) + try: + keys.sort() + except AttributeError: + pass + for key in keys: + child = self.db_layout_tree.AppendItem(item, str(key)) + self.db_layout_tree.SetPyData(child, data[key]) + self._set_child_icons(child, data[key]) + elif isinstance(data, (list, collections.UserList)): + self.txtData.Clear() + self.txtType.SetLabel(str(type(data))) + for d in data: + self.txtData.AppendText(str(type(d)) + ' --\n') + if hasattr(d, 'wx_str'): + self.txtData.AppendText(d.wx_str()) + else: + self.txtData.AppendText(str(d)) + self.txtData.AppendText('\n') + else: + self.txtType.SetLabel(str(type(data))) + # fmt = '%s\n-----\n%s' + if hasattr(data, 'wx_str'): + self.txtData.SetValue(data.wx_str()) + else: + self.txtData.SetValue(str(data)) + +# end of class ZODBFrame + +if __name__ == "__main__": + locale.setlocale(locale.LC_ALL, '') + wxviewdb = wx.PySimpleApp(0) + wx.InitAllImageHandlers() + frmZODB = ZODBFrame(None, -1, "") + wxviewdb.SetTopWindow(frmZODB) + frmZODB.Show() + wxviewdb.MainLoop() diff --git a/src/zcm.py b/src/zcm.py new file mode 100644 index 0000000..a61f2dd --- /dev/null +++ b/src/zcm.py @@ -0,0 +1,97 @@ +#!/usr/bin/python3.5 +# encoding: utf8 + +# Copyright (c) 2016 Andreas +# See LICENSE for details. + +""" ZODB context manager """ + +# from BTrees.IOBTree import IOBTree +# from BTrees.OOBTree import OOBTree +# from persistent import Persistent +# from persistent.list import PersistentList as PList +# from persistent.mapping import PersistentMapping as PDict + +from ZEO.ClientStorage import ClientStorage +from ZODB import DB +from ZODB.FileStorage import FileStorage + + +class ZDatabase(): + """ Provides a ZODB database context manager """ + + def __init__(self, uri, **kwargs): + self.storage = create_storage(uri) + self.db = DB(self.storage, **kwargs) + + def __enter__(self): + return self.db + + def __exit__(self, exc_type, exc_value, traceback): + self.db.close() + return False + + +class ZConnection(): + """ Provides a ZODB connection with auto-abort (default). + Provides a tuple of connection and root object: + with ZConnection(db) as (cx, root): + root.one = "ok" + ZConnection implements a connection context manager. + Transaction context managers in contrast do auto-commit: + a) with db.transaction() as connection, or + b) with cx.transaction_manager as transaction, or + c) with transaction.manager as transaction (for the thread-local transaction manager) + See also http://www.zodb.org/en/latest/guide/transactions-and-threading.html + """ + def __init__(self, db, auto_commit=False, transaction_manager=None): + self.db = db + self.auto_commit = auto_commit + self.transaction_manager = transaction_manager + self.cx = None + + def __enter__(self): + if self.transaction_manager: + self.cx = self.db.open(self.transaction_manager) + else: + self.cx = self.db.open() + return self.cx, self.cx.root() + + def __exit__(self, exc_type, exc_value, traceback): + if self.auto_commit: + self.cx.transaction_manager.commit() + self.cx.close() + return False + + +def create_storage(uri): + """ supported URIs + file://e:/workspaces/zeo/bots.fs + zeo://localhost:8001 + e:/workspaces/zeo/bots.fs + @see https://en.wikipedia.org/wiki/Uniform_Resource_Identifier + """ + if uri.startswith("file://"): + storage = FileStorage(uri[7:]) + elif uri.startswith("zeo://"): + addr, port = uri[6:].split(":") + # addr_ = addr.encode("ASCII") + storage = ClientStorage((addr, int(port))) + else: + storage = FileStorage(uri) + return storage + + +def database(uri): + """ convenience function for single thread, return one connection from the pool """ + storage = create_storage(uri) + db = DB(storage) + return db + + +def connection(db): + """ Convenience function for multi thread, returns + connection, transaction manager and root + """ + cx = db.open() + return cx, cx.root() diff --git a/test/test_zcm.py b/test/test_zcm.py new file mode 100644 index 0000000..7c37216 --- /dev/null +++ b/test/test_zcm.py @@ -0,0 +1,109 @@ +#!/usr/bin/python3.5 +# encoding: utf8 + +"""Test zodb database class""" + +from pprint import pprint +import unittest +import transaction +import ZODB.POSException + +import zcm + + +FILE = r"..\var\unittest.zodb" + + +class TestContext(unittest.TestCase): + + def test_sync(self): + print("sync") + db = zcm.database(FILE) + cx = db.open() + cx.root.one = 2 + print(cx.root.one) + with db.transaction() as cx2: + _ = cx2.transaction_manager + root = cx2.root() + root['one'] = 3 + transaction.abort() + print(cx.root.one) + cx.root.one = 4 + print(cx.root.one) + db.close() + + def test_transaction(self): + print("transaction") + db = zcm.database(FILE) + cx = db.open() + cx.root.one = 2 + print(cx.root.one) + with db.transaction() as cx2: + _ = cx2.transaction_manager + root = cx2.root() + root['one'] = 3 + print(cx.root.one) + cx.root.one = 4 + print(cx.root.one) + cx3 = db.open() + print(cx3.root.one) + db.close() + + def test_conflict(self): + db = zcm.database(FILE) + try: + cx = db.open() + cx.root.one = 2 + print(cx.root.one) + with db.transaction() as cx2: + _ = cx2.transaction_manager + root = cx2.root() + root['one'] = 3 + print(cx.root.one) + cx.root.one = 4 + print(cx.root.one) + with self.assertRaises(ZODB.POSException.ConflictError): + transaction.commit() + finally: + db.close() + db = zcm.database(FILE) + cx = db.open() + print(cx.root.one) + db.close() + + def test_connection(self): + db = zcm.database(FILE) + txm = transaction.TransactionManager() + cx3 = db.open(txm) + print(cx3.root.another) + cx3.root.another = "no tx" + print(cx3.root.another) + with txm: + cx3.root.another = "object" + print(cx3.root.another) + db.close() + + def test_default_txm(self): + print("default_txm") + db = zcm.database(FILE) + cx2, root = zcm.connection(db) + cx3 = db.open() + if cx2.transaction_manager is cx3.transaction_manager: + print("cx3 identity") + print(root.another) + with cx2.transaction_manager: + root.another = "more" + print(cx3.root.another) + with db.transaction() as cx4: + cx4.root.another = "other" + if cx4.transaction_manager == cx2.transaction_manager: + print("cx4 same") + else: + print("cx4 differs") + cx2.transaction_manager.abort() + print(root.another) + with cx3.transaction_manager: + cx3.root.another = "object" + print(root.another) + print(cx3.root.another) + db.close()