From 773d4dfe6cea00c2ae796b6ea2a8704f8f57dd8c Mon Sep 17 00:00:00 2001 From: Andreas Date: Wed, 24 Jun 2009 19:11:31 +0000 Subject: [PATCH] Initial checkin --HG-- branch : sandbox --- .project | 17 + .pydevproject | 12 + .../org.eclipse.ltk.core.refactoring.prefs | 3 + .settings/org.eclipse.mylyn.tasks.ui.prefs | 4 + .settings/org.eclipse.wst.validation.prefs | 6 + gui/bottier.py | 365 ++++++++++++++ mpl/fibionacci.py | 449 ++++++++++++++++++ mpl/fibionacci2.py | 444 +++++++++++++++++ mpl/mpl-blit.py | 174 +++++++ mpl/mpl-draw.py | 163 +++++++ mpl/mpl-embedded.py | 330 +++++++++++++ src/batch_rename.py | 62 +++ src/ftpget.py | 6 + src/histogram.py | 97 ++++ src/pricefeed.py | 35 ++ src/roimatrix.py | 94 ++++ src/wget_batch.py | 52 ++ 17 files changed, 2313 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.ltk.core.refactoring.prefs create mode 100644 .settings/org.eclipse.mylyn.tasks.ui.prefs create mode 100644 .settings/org.eclipse.wst.validation.prefs create mode 100644 gui/bottier.py create mode 100644 mpl/fibionacci.py create mode 100644 mpl/fibionacci2.py create mode 100644 mpl/mpl-blit.py create mode 100644 mpl/mpl-draw.py create mode 100644 mpl/mpl-embedded.py create mode 100644 src/batch_rename.py create mode 100644 src/ftpget.py create mode 100644 src/histogram.py create mode 100644 src/pricefeed.py create mode 100644 src/roimatrix.py create mode 100644 src/wget_batch.py diff --git a/.project b/.project new file mode 100644 index 0000000..0a3bdde --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + sandbox + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..756cdbd --- /dev/null +++ b/.pydevproject @@ -0,0 +1,12 @@ + + + + + +/sandbox/src +/sandbox/gui +/sandbox/mpl + +python 2.5 +Default + diff --git a/.settings/org.eclipse.ltk.core.refactoring.prefs b/.settings/org.eclipse.ltk.core.refactoring.prefs new file mode 100644 index 0000000..52fc620 --- /dev/null +++ b/.settings/org.eclipse.ltk.core.refactoring.prefs @@ -0,0 +1,3 @@ +#Mon Feb 11 22:12:56 CET 2008 +eclipse.preferences.version=1 +org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false diff --git a/.settings/org.eclipse.mylyn.tasks.ui.prefs b/.settings/org.eclipse.mylyn.tasks.ui.prefs new file mode 100644 index 0000000..7503005 --- /dev/null +++ b/.settings/org.eclipse.mylyn.tasks.ui.prefs @@ -0,0 +1,4 @@ +#Mon Feb 11 22:12:56 CET 2008 +eclipse.preferences.version=1 +project.repository.kind=trac +project.repository.url=http\://localhost\:8080/xmlrpc diff --git a/.settings/org.eclipse.wst.validation.prefs b/.settings/org.eclipse.wst.validation.prefs new file mode 100644 index 0000000..2636ca4 --- /dev/null +++ b/.settings/org.eclipse.wst.validation.prefs @@ -0,0 +1,6 @@ +#Mon Feb 11 22:12:56 CET 2008 +DELEGATES_PREFERENCE=delegateValidatorListorg.eclipse.wst.xsd.core.internal.validation.eclipse.XSDDelegatingValidator\=org.eclipse.wst.xsd.core.internal.validation.eclipse.Validator; +USER_BUILD_PREFERENCE=enabledBuildValidatorListorg.eclipse.wst.xsd.core.internal.validation.eclipse.XSDDelegatingValidator;org.eclipse.wst.dtd.core.internal.validation.eclipse.Validator;org.eclipse.wst.xml.core.internal.validation.eclipse.Validator; +USER_MANUAL_PREFERENCE=enabledManualValidatorListorg.eclipse.wst.xsd.core.internal.validation.eclipse.XSDDelegatingValidator;org.eclipse.wst.dtd.core.internal.validation.eclipse.Validator;org.eclipse.wst.xml.core.internal.validation.eclipse.Validator; +USER_PREFERENCE=overrideGlobalPreferencesfalse +eclipse.preferences.version=1 diff --git a/gui/bottier.py b/gui/bottier.py new file mode 100644 index 0000000..7469f4f --- /dev/null +++ b/gui/bottier.py @@ -0,0 +1,365 @@ +# Copyright (c) 2008 Andreas Balogh +# See LICENSE for details. + +""" decode ASCII greyscale """ + +# system imports + +import Tkinter as Tk +import logging +from StringIO import StringIO +from PIL import Image +from PIL import ImageTk + +# local imports + +# constants + +# globals + +LOG = logging.getLogger() + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s.%(msecs)03i %(levelname).4s %(process)d:%(thread)d %(message)s', + datefmt='%H:%M:%S') + +CODE = "#MWmBEKNQkAbqXHgUFRhdwp8&aDGVZ9utO03CYvlrI=i<7+?}(*[/|;!^-~`,:. " + +def main(): + tkis = [] + # build gui + root = Tk.Tk() + root.title("bottier") + bu1 = Tk.Button(root, text='Quit', command= root.quit) + bu1.pack(side = Tk.BOTTOM, padx = 5, pady = 5) + for s in (PIC1, PIC2, PIC3, PIC4, PIC5): + im = ascii2img(s) + tki = ImageTk.PhotoImage(im, gamma = 2) + tkis.append(tki) + la = Tk.Label(root, image = tki) + la.pack(side = Tk.RIGHT, padx = 5, pady = 5) + + root.mainloop() + root.destroy() + +def ascii2img(s): + # create the image + # lines = StringIO(s).readlines() + # height = len(lines) + # width = len(lines[0]) + # im = Image.new("L", (width, height)) + im = Image.new("L", (100, 100)) + for y, line in enumerate(StringIO(s)): + for x, c in enumerate(line): + im.putpixel((x, y), CODE.find(c) * 4) + bbox = im.getbbox() + bbim = im.crop(bbox) + imx2 = bbim.resize((bbox[2]*2, bbox[3]*2), Image.ANTIALIAS) + return imx2 + +PIC1 = """\ +BEmmmBBBBBNNEBBEEKBBHd&aDGapHNBEBBBBEKBEmBBEBBKgdp8wdRbEBKEEEEENKQNAAbbXUp8aD9 +BEBBBEKENNNEBEEKEEmAd8DZVG&RkEBmBBEEKBBBmmmmBBmbh&Da&wFQBmmmBmBmmmBWmmmBmmmBmm +mBBBBBEKNKKKKKKENBkh8VVZGDwXNEBBBEEBBBBBmWmmmmmEgwaDGDpFKBBmmmmmmWmWWmWWWWWmmW +EEBBKBEKNNKNEEEQEER&VZ99V8gNKBEBBEBEBBBmmmmmmmmWqh&DVGawqmEmBmmmmmBWmWWmWWMWWM +KEBKEEKNKKKKKNNNBqG9OOtuGhAKEEEEBBBBBBBBBmmBBBmmEgwaDaa8RNBBBmBBBmmmmWWmWMWWWM +BBBBENKQNKENNQNNmRu0003taXNKKEKEBEEBEBmmBBBmmBBmmNF8aDDawXKmWBBWBmmmMWWmWWWMMM +EKEBKNkQNKNNNNQNBRO333YO&bQNKKKKBBBmWMWWMWWmmmBmmWAR8VGG&hbBmWWmmmWWWWMMMMMMMM +NKEENAAkNKNkQNkKQ&3YYvvO&AKEEEBBmWWMMMMW#MMMMMMMMMWEgpGGDawbBBmBmBWmmWWWMWMWMM +NKEKkAkkKNKkQQkKXZClrIv3DbNEWMMMMM#MMM#MM#MMM#MMMMMMmNgZZGVwEmBmmmWmmmmmWWWMM# +QNQkkQQkNQQkkAkERtYr=Il3VHKWMMMMMMMMMM#M#MMWMMMMMMMMMEAdZVZREEmBBmmWmmmmmWWWMW +kbkbAkQkNQQAkkQEpOv=i=rY9hBWWMWMMMMMMWMMBABmWMMWWWWMWWN899VdKEEBBmmmWmBBmmmWWW +AbbbbkkkNQbbAQQNDCrii=Iv9dKmMMMMMMMMWWWmBXAKEmKghFQmMMBqVt9&QEEBBmBmmBBmmmmmmm +bqqXXAkQNAqqXbQAG3I< 0) + self.bias = bias + self.low0 = None + self.high0 = None + self.prev_lohi = NONE + self.lohis = [ ] + self.lows = [ ] + self.highs = [ ] + + def __call__(self, tick): + """Add extended tick to the max min parser. + + @param tick: The value of the current tick. + @type tick: tuple(cdt, last) + + @return: 1. Tick if new max min has been detected, + 2. None otherwise. + """ + n, cdt, last = tick + res = None + # automatic initialisation + if self.low0 is None: + self.low0 = tick + self.lows.append((n, cdt, last - 1)) + if self.high0 is None: + self.high0 = tick + self.highs.append((n, cdt, last + 1)) + if last > self.high0[2]: + self.high0 = tick + if self.prev_lohi == NONE: + if self.high0[2] > self.low0[2] + self.bias: + res = self.high0 + self.low0 = self.high0 + self.lows.append(self.high0) + self.lohis.append(self.high0) + self.prev_lohi = HIGH + if last < self.low0[2]: + self.low0 = tick + if self.prev_lohi == NONE: + if self.low0[2] < self.high0[2] - self.bias: + res = self.low0 + self.high0 = self.low0 + self.lows.append(self.low0) + self.lohis.append(self.low0) + self.prev_lohi = LOW + if self.high0[1] < cdt - ONE_MINUTE and \ + ((self.prev_lohi == LOW and \ + self.high0[2] > self.lows[-1][2] + self.bias) or + (self.prev_lohi == HIGH and \ + self.high0[2] > self.highs[-1][2])): + res = self.high0 + self.low0 = self.high0 + self.highs.append(self.high0) + self.lohis.append(self.high0) + self.prev_lohi = HIGH + if self.low0[1] < cdt - ONE_MINUTE and \ + ((self.prev_lohi == LOW and \ + self.low0[2] < self.lows[-1][2]) or + (self.prev_lohi == HIGH and \ + self.low0[2] < self.highs[-1][2] - self.bias)): + res = self.low0 + self.high0 = self.low0 + self.lows.append(self.low0) + self.lohis.append(self.low0) + self.prev_lohi = LOW + if res: + return (self.prev_lohi, res) + else: + return None + + +class Acp: + """Always correct predictor""" + def __init__(self, lows, highs): + self.lows = lows + self.highs = highs + + def __call__(self, tick): + """Always correct predictor. + + Requires previous run of DelayedAcp to determine lows and highs. + + @param tick: The value of the current tick. + @type tick: tuple(n, cdt, last) + + @return: 1. Tick if new max min has been detected, + 2. None otherwise. + """ + n, cdt, last = tick + res = None + return res + + +def find_lows_highs(xs, ys): + dacp = DelayedAcp(10) + for tick in zip(range(len(xs)), xs, ys): + dacp(tick) + return dacp.lows, dacp.highs + + +class DelayedAcp: + """Time series max & min detector.""" + def __init__(self, bias): + assert(bias > 0) + self.bias = bias + self.trend = None + self.mm0 = None + self.lohis = [ ] + self.lows = [ ] + self.highs = [ ] + + def __call__(self, tick): + """Add extended tick to the max min parser. + + @param tick: The value of the current tick. + @type tick: tuple(n, cdt, last) + + @return: 1. Tick if new max min has been detected, + 2. None otherwise. + """ + n, cdt, last = tick + res = None + # automatic initialisation + if self.mm0 is None: + # initalise water mark + self.mm0 = tick + res = self.mm0 + self.lows = [(n, cdt, last - 1)] + self.highs = [(n, cdt, last + 1)] + else: + # initalise trend until price has changed + if self.trend is None or self.trend == 0: + self.trend = cmp(last, self.mm0[2]) + # check for max + if self.trend > 0: + if last > self.mm0[2]: + self.mm0 = tick + if last < self.mm0[2] - self.bias: + self.lohis.append(self.mm0) + self.highs.append(self.mm0) + res = self.mm0 + # revert trend & water mark + self.mm0 = tick + self.trend = -1 + # check for min + if self.trend < 0: + if last < self.mm0[2]: + self.mm0 = tick + if last > self.mm0[2] + self.bias: + self.lohis.append(self.mm0) + self.lows.append(self.mm0) + res = self.mm0 + # revert trend & water mark + self.mm0 = tick + self.trend = +1 + return (cmp(self.trend, 0), res) + + +class Main: + def __init__(self): + warnings.simplefilter("default", np.RankWarning) + self.ylow = None + self.yhigh = None + self.advance_count = 1 + + self.root = Tk.Tk() + self.root.wm_title("Embedding in TK") + + # create plot + fig = plt.figure() + self.ax1 = fig.add_subplot(211) # ticks + self.ax2 = fig.add_subplot(212) # diff from polyfit + # ax3 = fig.add_subplot(313) # cash + + self.ax1.set_ylabel("ticks") + self.ax2.set_ylabel("polyfit diff") + # ax3.set_ylabel("cash") + + major_fmt = mdates.DateFormatter('%H:%M:%S') + self.ax1.xaxis.set_major_formatter(major_fmt) + self.ax1.format_xdata = major_fmt + self.ax1.format_ydata = lambda x: '%1.2f' % x + self.ax1.grid(True) + + self.ax2.xaxis.set_major_formatter(major_fmt) + self.ax2.format_xdata = major_fmt + self.ax2.format_ydata = lambda x: '%1.2f' % x + self.ax2.grid(True) + + # rotates and right aligns the x labels, and moves the bottom of the + # axes up to make room for them + fig.autofmt_xdate() + + # create artists + LOG.debug("Loading ticks...") + self.x, self.y = tdl(datetime.datetime(2009, 6, 3)) + LOG.debug("Ticks loaded.") + lows, highs = find_lows_highs(self.x, self.y) + + self.mmh = Lohi(10) + + self.w0 = 0 + self.wd = 1000 + self.low_high_crs = 0 + xr, yr = self.tick_window(self.w0, self.wd) + fit = np.average(yr) + + self.tl, = self.ax1.plot_date(xr, yr, '-') + self.fl, = self.ax1.plot_date(xr, (fit,) * len(xr), 'k--') + self.mh, = self.ax1.plot_date(xr, (yr[0],) * len(xr), 'g:') + self.ml, = self.ax1.plot_date(xr, (yr[0],) * len(xr), 'r:') + self.him, = self.ax1.plot_date([x for n, x, y in lows], [y for n, x, y in lows], 'go') + self.lom, = self.ax1.plot_date([x for n, x, y in highs], [y for n, x, y in highs], 'ro') + + self.dl, = self.ax2.plot_date(xr, (0,) * len(xr), '-') + + self.set_axis(xr, yr) + + # embed canvas in Tk + self.canvas = FigureCanvasTkAgg(fig, master=self.root) + self.canvas.draw() + self.canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=Tk.TRUE) + + # toolbar = NavigationToolbar2TkAgg( self.canvas, self.root ) + # toolbar.update() + # self.canvas._tkself.canvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) + + fr1 = Tk.Frame(master=self.root) + bu1 = Tk.Button(master=fr1, text='Quit', command=self.root.quit) + bu2 = Tk.Button(master=fr1, text='Stop', command=self.stop) + bu3 = Tk.Button(master=fr1, text='Resume', command=self.resume) + bu4 = Tk.Button(master=fr1, text='1x', command=self.times_one) + bu5 = Tk.Button(master=fr1, text='5x', command=self.times_five) + bu6 = Tk.Button(master=fr1, text='10x', command=self.times_ten) + bu1.pack(side=Tk.RIGHT, padx=5, pady=5) + bu6.pack(side=Tk.RIGHT, padx=5, pady=5) + bu5.pack(side=Tk.RIGHT, padx=5, pady=5) + bu4.pack(side=Tk.RIGHT, padx=5, pady=5) + bu2.pack(side=Tk.RIGHT, padx=5, pady=5) + bu3.pack(side=Tk.RIGHT, padx=5, pady=5) + fr1.pack(side=Tk.BOTTOM) + + + def animate(self): + self.w0 += self.advance_count + # prepare timeline window + xr, yr = self.tick_window(self.w0, self.wd) + while self.low_high_crs < self.w0 + self.wd: + self.mark_low_high(self.low_high_crs) + self.low_high_crs += 1 + # build polynomial fit + mms = self.mmh.lohis + if len(mms) > 1: + mx = [ (x - int(x)) * 86400 for x in xr ] + my = yr + polyval = np.polyfit(mx, my, 10) + fit = np.polyval(polyval, mx) + self.fl.set_data(xr, fit) + # calc diff + self.dl.set_data(xr, yr - fit) + maxs = self.mmh.highs + if len(maxs) > 1: + n, x1, y1 = maxs[-2] + n, x2, y2 = maxs[-1] + x3 = xr[-1] + polyfit = np.polyfit((x1, x2), (y1, y2), 1) + y3 = np.polyval(polyfit, x3) + self.mh.set_data((x1, x2, x3), (y1, y2, y3)) + mins = self.mmh.lows + if len(mins) > 1: + n, x1, y1 = mins[-2] + n, x2, y2 = mins[-1] + x3 = xr[-1] + polyfit = np.polyfit((x1, x2), (y1, y2), 1) + y3 = np.polyval(polyfit, x3) + self.ml.set_data((x1, x2, x3), (y1, y2, y3)) + # update tick line + self.tl.set_data(xr, yr) + # update axis + self.set_axis(xr, yr) + self.canvas.draw() + if self.w0 < len(self.x) - self.wd - 1: + self.after_id = self.root.after(10, self.animate) + + def set_axis(self, xr, yr, bias=50): + if self.ylow is None: + self.ylow = yr[0] - bias / 2 + self.yhigh = yr[0] + bias / 2 + for y in yr: + if y < self.ylow: + self.ylow = y + self.yhigh = self.ylow + bias + if y > self.yhigh: + self.yhigh = y + self.ylow = self.yhigh - bias + self.ax1.axis([xr[0], xr[-1], self.ylow, self.yhigh]) + self.ax2.axis([xr[0], xr[-1], -25, +25]) + + def tick_window(self, w0, wd=1000): + return (self.x[w0:w0 + wd], self.y[w0:w0 + wd]) + + def mark_low_high(self, n): + x = self.x + y = self.y + rc = self.mmh((n, x[n], y[n])) + if rc: + lohi, tick = rc + nlh, xlh, ylh = tick + if lohi < 0: + # low + self.ax1.annotate('low', + xy=(x[nlh], y[nlh]), + xytext=(x[n], y[nlh]), + arrowprops=dict(facecolor='red', + frac=0.3, + shrink=0.1)) + elif lohi > 0: + # high + self.ax1.annotate('high', + xy=(x[nlh], y[nlh]), + xytext=(x[n], y[nlh]), + arrowprops=dict(facecolor='green', + frac=0.3, + shrink=0.1)) + + def stop(self): + if self.after_id: + self.root.after_cancel(self.after_id) + self.after_id = None + + def resume(self): + if self.after_id is None: + self.after_id = self.root.after(10, self.animate) + + def times_one(self): + self.advance_count = 1 + + def times_five(self): + self.advance_count = 5 + + def times_ten(self): + self.advance_count = 10 + + def run(self): + self.root.after(500, self.animate) + self.root.mainloop() + self.root.destroy() + + +if __name__ == "__main__": + app = Main() + app.run() diff --git a/mpl/fibionacci2.py b/mpl/fibionacci2.py new file mode 100644 index 0000000..27b27f7 --- /dev/null +++ b/mpl/fibionacci2.py @@ -0,0 +1,444 @@ +# Copyright (c) 2008 Andreas Balogh +# See LICENSE for details. + +""" animated drawing of ticks + + using self.canvas embedded in Tk application + + Fibionacci retracements of 61.8, 50.0, 38.2, 23.6 % of min and max +""" + +# system imports + +import datetime +import os +import re +import logging +import warnings +import math + +import Tkinter as Tk +import numpy as np + +import matplotlib +matplotlib.use('TkAgg') + +from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +from matplotlib.dates import date2num + +# local imports + +# constants + +ONE_MINUTE = 60. / 86400. +LOW, NONE, HIGH = range(-1, 2) + +# globals + +LOG = logging.getLogger() + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s.%(msecs)03i %(levelname).4s %(process)d:%(thread)d %(message)s', + datefmt='%H:%M:%S') + +MDF_REO = re.compile("(..):(..):(..)\.*(\d+)*") + + +def tdl(tick_date): + """ returns a list of tick tuples (cdt, last) for specified day """ + fiid = "846900" + year = tick_date.strftime("%Y") + yyyymmdd = tick_date.strftime("%Y%m%d") + filename = "%s.csv" % (fiid) + filepath = os.path.join("c:\\rttrd-prd-var\\consors-mdf\\data", year, yyyymmdd, filename) + x = [ ] + y = [ ] + fh = open(filepath, "r") + try: + prev_last = "" + for line in fh: + flds = line.split(",") + # determine file version + if flds[2] == "LAST": + last = float(flds[3]) + vol = float(flds[4]) + else: + last = float(flds[4]) + vol = 0.0 + # skip ticks with same last price + if prev_last == last: + continue + else: + prev_last = last + # parse time + mobj = MDF_REO.match(flds[0]) + if mobj is None: + raise ValueError("no match for [%s]" % (flds[0],)) + (hh, mm, ss, ms) = mobj.groups() + if ms: + c_time = datetime.time(int(hh), int(mm), int(ss), int(ms) * 1000) + else: + c_time = datetime.time(int(hh), int(mm), int(ss)) + cdt = datetime.datetime.combine(tick_date, c_time) + x.append(date2num(cdt)) + y.append(last) + finally: + fh.close() + # throw away first line of file (close price from previous day) + del x[0] + del y[0] + return (x, y) + +def num2sod(x): + frac, integ = math.modf(x) + return frac * 86400 + +class Lohi: + """Time series online low and high detector.""" + def __init__(self, bias): + assert(bias > 0) + self.bias = bias + self.low0 = None + self.high0 = None + self.prev_lohi = NONE + self.lohis = [ ] + self.lows = [ ] + self.highs = [ ] + + def __call__(self, tick): + """Add extended tick to the max min parser. + + @param tick: The value of the current tick. + @type tick: tuple(cdt, last) + + @return: 1. Tick if new max min has been detected, + 2. None otherwise. + """ + n, cdt, last = tick + res = None + # automatic initialisation + if self.low0 is None: + self.low0 = tick + self.lows.append((n, cdt, last - 1)) + if self.high0 is None: + self.high0 = tick + self.highs.append((n, cdt, last + 1)) + if last > self.high0[2]: + self.high0 = tick + if self.prev_lohi == NONE: + if self.high0[2] > self.low0[2] + self.bias: + res = self.high0 + self.low0 = self.high0 + self.lows.append(self.high0) + self.lohis.append(self.high0) + self.prev_lohi = HIGH + if last < self.low0[2]: + self.low0 = tick + if self.prev_lohi == NONE: + if self.low0[2] < self.high0[2] - self.bias: + res = self.low0 + self.high0 = self.low0 + self.lows.append(self.low0) + self.lohis.append(self.low0) + self.prev_lohi = LOW + if self.high0[1] < cdt - ONE_MINUTE and \ + ((self.prev_lohi == LOW and \ + self.high0[2] > self.lows[-1][2] + self.bias) or + (self.prev_lohi == HIGH and \ + self.high0[2] > self.highs[-1][2])): + res = self.high0 + self.low0 = self.high0 + self.highs.append(self.high0) + self.lohis.append(self.high0) + self.prev_lohi = HIGH + if self.low0[1] < cdt - ONE_MINUTE and \ + ((self.prev_lohi == LOW and \ + self.low0[2] < self.lows[-1][2]) or + (self.prev_lohi == HIGH and \ + self.low0[2] < self.highs[-1][2] - self.bias)): + res = self.low0 + self.high0 = self.low0 + self.lows.append(self.low0) + self.lohis.append(self.low0) + self.prev_lohi = LOW + if res: + return (self.prev_lohi, res) + else: + return None + + +class Acp: + """Always correct predictor""" + def __init__(self, lows, highs): + self.lows = lows + self.highs = highs + + def __call__(self, tick): + """Always correct predictor. + + Requires previous run of DelayedAcp to determine lows and highs. + + @param tick: The value of the current tick. + @type tick: tuple(n, cdt, last) + + @return: 1. Tick if new max min has been detected, + 2. None otherwise. + """ + n, cdt, last = tick + res = None + return res + + +def find_lows_highs(xs, ys): + dacp = DelayedAcp(10) + for tick in zip(range(len(xs)), xs, ys): + dacp(tick) + return dacp.lows, dacp.highs + + +class DelayedAcp: + """Time series max & min detector.""" + def __init__(self, bias): + assert(bias > 0) + self.bias = bias + self.trend = None + self.mm0 = None + self.lohis = [ ] + self.lows = [ ] + self.highs = [ ] + + def __call__(self, tick): + """Add extended tick to the max min parser. + + @param tick: The value of the current tick. + @type tick: tuple(n, cdt, last) + + @return: 1. Tick if new max min has been detected, + 2. None otherwise. + """ + n, cdt, last = tick + res = None + # automatic initialisation + if self.mm0 is None: + # initalise water mark + self.mm0 = tick + res = self.mm0 + self.lows = [(n, cdt, last - 1)] + self.highs = [(n, cdt, last + 1)] + else: + # initalise trend until price has changed + if self.trend is None or self.trend == 0: + self.trend = cmp(last, self.mm0[2]) + # check for max + if self.trend > 0: + if last > self.mm0[2]: + self.mm0 = tick + if last < self.mm0[2] - self.bias: + self.lohis.append(self.mm0) + self.highs.append(self.mm0) + res = self.mm0 + # revert trend & water mark + self.mm0 = tick + self.trend = -1 + # check for min + if self.trend < 0: + if last < self.mm0[2]: + self.mm0 = tick + if last > self.mm0[2] + self.bias: + self.lohis.append(self.mm0) + self.lows.append(self.mm0) + res = self.mm0 + # revert trend & water mark + self.mm0 = tick + self.trend = +1 + return (cmp(self.trend, 0), res) + + +class Main: + def __init__(self): + warnings.simplefilter("default", np.RankWarning) + self.ylow = None + self.yhigh = None + self.advance_count = 1 + + self.root = Tk.Tk() + self.root.wm_title("Embedding in TK") + + # create plot + fig = plt.figure() + self.ax1 = fig.add_subplot(211) # ticks + self.ax2 = fig.add_subplot(212) # diff from polyfit + # ax3 = fig.add_subplot(313) # cash + + self.ax1.set_ylabel("ticks") + self.ax2.set_ylabel("polyfit diff") + # ax3.set_ylabel("cash") + + major_fmt = mdates.DateFormatter('%H:%M:%S') + self.ax1.xaxis.set_major_formatter(major_fmt) + self.ax1.format_xdata = major_fmt + self.ax1.format_ydata = lambda x: '%1.2f' % x + self.ax1.grid(True) + + self.ax2.xaxis.set_major_formatter(major_fmt) + self.ax2.format_xdata = major_fmt + self.ax2.format_ydata = lambda x: '%1.2f' % x + self.ax2.grid(True) + + # rotates and right aligns the x labels, and moves the bottom of the + # axes up to make room for them + fig.autofmt_xdate() + + # create artists + LOG.debug("Loading ticks...") + self.x, self.y = tdl(datetime.datetime(2009, 6, 3)) + LOG.debug("Ticks loaded.") + lows, highs = find_lows_highs(self.x, self.y) + + self.mmh = Lohi(10) + + self.w0 = 0 + self.wd = 1000 + self.low_high_crs = 0 + xr, yr = self.tick_window(self.w0, self.wd) + fit = np.average(yr) + + self.tl, = self.ax1.plot_date(xr, yr, '-') + self.fl, = self.ax1.plot_date(xr, (fit,) * len(xr), 'k--') + self.mh, = self.ax1.plot_date(xr, (yr[0],) * len(xr), 'g:') + self.ml, = self.ax1.plot_date(xr, (yr[0],) * len(xr), 'r:') + self.him, = self.ax1.plot_date([x for n, x, y in lows], [y for n, x, y in lows], 'go') + self.lom, = self.ax1.plot_date([x for n, x, y in highs], [y for n, x, y in highs], 'ro') + + self.dl, = self.ax2.plot_date(xr, (0,) * len(xr), '-') + + self.set_axis(xr, yr) + + # embed canvas in Tk + self.canvas = FigureCanvasTkAgg(fig, master=self.root) + self.canvas.draw() + self.canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=Tk.TRUE) + + # toolbar = NavigationToolbar2TkAgg( self.canvas, self.root ) + # toolbar.update() + # self.canvas._tkself.canvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) + + fr1 = Tk.Frame(master=self.root) + bu1 = Tk.Button(master=fr1, text='Quit', command=self.root.quit) + bu2 = Tk.Button(master=fr1, text='Stop', command=self.stop) + bu3 = Tk.Button(master=fr1, text='Resume', command=self.resume) + bu4 = Tk.Button(master=fr1, text='1x', command=self.times_one) + bu5 = Tk.Button(master=fr1, text='5x', command=self.times_five) + bu6 = Tk.Button(master=fr1, text='10x', command=self.times_ten) + bu1.pack(side=Tk.RIGHT, padx=5, pady=5) + bu6.pack(side=Tk.RIGHT, padx=5, pady=5) + bu5.pack(side=Tk.RIGHT, padx=5, pady=5) + bu4.pack(side=Tk.RIGHT, padx=5, pady=5) + bu2.pack(side=Tk.RIGHT, padx=5, pady=5) + bu3.pack(side=Tk.RIGHT, padx=5, pady=5) + fr1.pack(side=Tk.BOTTOM) + + + def animate(self): + self.w0 += self.advance_count + # prepare timeline window + xr, yr = self.tick_window(self.w0, self.wd) + while self.low_high_crs < self.w0 + self.wd: + self.mark_low_high(self.low_high_crs) + self.low_high_crs += 1 + # build polynomial fit + mms = self.mmh.lohis + if len(mms) > 1: + mx = [ num2sod(x) for x in xr ] + my = yr + polyval = np.polyfit(mx, my, 10) + fit = np.polyval(polyval, mx) + self.fl.set_data(xr, fit) + # calc diff + self.dl.set_data(xr, yr - fit) + highs = self.mmh.highs + if len(highs) >= 4: + coefs = np.polyfit([num2sod(x) for n, x, y in highs[-4:]], [y for n, x, y in highs[-4:]], 3) + self.mh.set_data(xr, [np.polyval(coefs, num2sod(x)) for x in xr]) + lows = self.mmh.lows + if len(lows) >= 4: + coefs = np.polyfit([num2sod(x) for n, x, y in lows[-4:]], [y for n, x, y in lows[-4:]], 3) + self.ml.set_data(xr, [np.polyval(coefs, num2sod(x)) for x in xr]) + # update tick line + self.tl.set_data(xr, yr) + # update axis + self.set_axis(xr, yr) + self.canvas.draw() + if self.w0 < len(self.x) - self.wd - 1: + self.after_id = self.root.after(10, self.animate) + + def set_axis(self, xr, yr, bias=50): + if self.ylow is None: + self.ylow = yr[0] - bias / 2 + self.yhigh = yr[0] + bias / 2 + for y in yr: + if y < self.ylow: + self.ylow = y + self.yhigh = self.ylow + bias + if y > self.yhigh: + self.yhigh = y + self.ylow = self.yhigh - bias + self.ax1.axis([xr[0], xr[-1], self.ylow, self.yhigh]) + self.ax2.axis([xr[0], xr[-1], -25, +25]) + + def tick_window(self, w0, wd = 1000): + return (self.x[w0:w0 + wd], self.y[w0:w0 + wd]) + + def mark_low_high(self, n): + x = self.x + y = self.y + rc = self.mmh((n, x[n], y[n])) + if rc: + lohi, tick = rc + nlh, xlh, ylh = tick + if lohi < 0: + # low + self.ax1.annotate('low', + xy=(x[nlh], y[nlh]), + xytext=(x[n], y[nlh]), + arrowprops=dict(facecolor='red', + frac=0.3, + shrink=0.1)) + elif lohi > 0: + # high + self.ax1.annotate('high', + xy=(x[nlh], y[nlh]), + xytext=(x[n], y[nlh]), + arrowprops=dict(facecolor='green', + frac=0.3, + shrink=0.1)) + + def stop(self): + if self.after_id: + self.root.after_cancel(self.after_id) + self.after_id = None + + def resume(self): + if self.after_id is None: + self.after_id = self.root.after(10, self.animate) + + def times_one(self): + self.advance_count = 1 + + def times_five(self): + self.advance_count = 5 + + def times_ten(self): + self.advance_count = 10 + + def run(self): + self.root.after(500, self.animate) + self.root.mainloop() + self.root.destroy() + + +if __name__ == "__main__": + app = Main() + app.run() diff --git a/mpl/mpl-blit.py b/mpl/mpl-blit.py new file mode 100644 index 0000000..615e7a8 --- /dev/null +++ b/mpl/mpl-blit.py @@ -0,0 +1,174 @@ +# Copyright (c) 2009 Andreas Balogh +# See LICENSE for details. + +""" animated drawing of ticks + + using blit method +""" + +# system imports + +import Tkinter as tk +import datetime +import os +import re +import logging +import sys + +import matplotlib.pyplot as plt +from matplotlib.dates import date2num +import matplotlib.dates as mdates +import numpy as np + +# local imports + +# constants + +# globals + +LOG = logging.getLogger() + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s.%(msecs)03i %(levelname).4s %(process)d:%(thread)d %(message)s', + datefmt='%H:%M:%S') + +MDF_REO = re.compile("(..):(..):(..)\.*(\d+)*") + +def tdl(tick_date): + """ returns a list of tick tuples (cdt, last) for specified day """ + fiid = "846900" + year = tick_date.strftime("%Y") + yyyymmdd = tick_date.strftime("%Y%m%d") + filename = "%s.csv" % (fiid) + filepath = os.path.join("d:\\rttrd-prd-var\\consors-mdf\\data", year, yyyymmdd, filename) + x = [ ] + y = [ ] + fh = open(filepath, "r") + try: + prev_last = "" + for line in fh: + flds = line.split(",") + # determine file version + if flds[2] == "LAST": + last = float(flds[3]) + vol = float(flds[4]) + else: + last = float(flds[4]) + vol = 0.0 + # skip ticks with same last price + if prev_last == last: + continue + else: + prev_last = last + # parse time + mobj = MDF_REO.match(flds[0]) + if mobj is None: + raise ValueError("no match for [%s]" % (flds[0],)) + (hh, mm, ss, ms) = mobj.groups() + if ms: + c_time = datetime.time(int(hh), int(mm), int(ss), int(ms) * 1000) + else: + c_time = datetime.time(int(hh), int(mm), int(ss)) + cdt = datetime.datetime.combine(tick_date, c_time) + x.append(date2num(cdt)) + y.append(last) + finally: + fh.close() + # throw away first line of file (close price from previous day) + del x[0] + del y[0] + return (x, y) + + +def main(): + LOG.debug("Loading ticks...") + x, y = tdl(datetime.datetime(2009,6,3)) + LOG.debug("Ticks loaded.") + + fig = plt.figure() + ax1 = fig.add_subplot(311) # ticks + canvas = ax1.figure.canvas + # ax2 = fig.add_subplot(312) # gearing + # ax3 = fig.add_subplot(313) # cash + + ax1.set_ylabel("ticks") + # ax2.set_ylabel("gearing") + # ax3.set_ylabel("cash") + + xr = [x[0]] * 500 + yr = [y[0]] * 500 + + line, = ax1.plot_date(xr, yr, '-', xdate = True) + major_fmt = mdates.DateFormatter('%H:%M:%S') + ax1.xaxis.set_major_formatter(major_fmt) + # ax1.axis('tight') + + # format the coords message box + def price(x): return '%1.2f'%x + # ax.format_xdata = mdates.DateFormatter('%Y-%m-%d') + ax1.format_xdata = mdates.DateFormatter('%H:%M:%S') + ax1.format_ydata = price + ax1.grid(True) + + # rotates and right aligns the x labels, and moves the bottom of the + # axes up to make room for them + fig.autofmt_xdate() + + background = canvas.copy_from_bbox(ax1.bbox) + + def animate(): + w = 1000 + bias = 10 + + ymin = min(y[0:100]) + ymax = max(y[0:100]) + + low = ymin - bias + high = ymax + bias + trend = 0 + + for i in range(0, len(x)-w): + # restore the clean slate background + canvas.restore_region(background) + # prepare timeline window + xr = x[i:i+w] + yr = y[i:i+w] + # update line + line.set_xdata(xr) + line.set_ydata(yr) + # determine y axis + if yr[-1] > ymax: + ymax = yr[-1] + if ymax - 50 < min(yr): + ymin = ymax - 50 + if yr[-1] < ymin: + ymin = yr[-1] + if ymin + 50 > max(yr): + ymax = ymin + 50 + ax1.axis([xr[0], xr[-1], ymin, ymax]) + # check low high and annotate + if i > 0 and i % 150 == 0: + ax1.annotate('low/high', + xy = (xr[-10], yr[-10]), + xytext = (xr[-5], yr[-10] + 10), + arrowprops = dict(facecolor = 'black', + frac = 0.2, + headwidth = 10, + linewidth = 0.1, + shrink = 0.05)) + + # just draw the animated artist + # FIXME: redraw all items in graph + ax1.draw_artist(line) + # just redraw the axes rectangle + canvas.blit(ax1.bbox) + + + + root = fig.canvas.manager.window + root.after(100, animate) + plt.show() + + +if __name__ == "__main__": + main() diff --git a/mpl/mpl-draw.py b/mpl/mpl-draw.py new file mode 100644 index 0000000..3f139ee --- /dev/null +++ b/mpl/mpl-draw.py @@ -0,0 +1,163 @@ +# Copyright (c) 2008 Andreas Balogh +# See LICENSE for details. + +""" animated drawing of ticks + + using draw method +""" + +# system imports + +import Tkinter as tk +import datetime +import os +import re +import logging +import sys + +import matplotlib.pyplot as plt +from matplotlib.dates import date2num +import matplotlib.dates as mdates +import numpy as np + +# local imports + +# constants + +# globals + +LOG = logging.getLogger() + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s.%(msecs)03i %(levelname).4s %(process)d:%(thread)d %(message)s', + datefmt='%H:%M:%S') + +MDF_REO = re.compile("(..):(..):(..)\.*(\d+)*") + +def tdl(tick_date): + """ returns a list of tick tuples (cdt, last) for specified day """ + fiid = "846900" + year = tick_date.strftime("%Y") + yyyymmdd = tick_date.strftime("%Y%m%d") + filename = "%s.csv" % (fiid) + filepath = os.path.join("d:\\rttrd-prd-var\\consors-mdf\\data", year, yyyymmdd, filename) + x = [ ] + y = [ ] + fh = open(filepath, "r") + try: + prev_last = "" + for line in fh: + flds = line.split(",") + # determine file version + if flds[2] == "LAST": + last = float(flds[3]) + vol = float(flds[4]) + else: + last = float(flds[4]) + vol = 0.0 + # skip ticks with same last price + if prev_last == last: + continue + else: + prev_last = last + # parse time + mobj = MDF_REO.match(flds[0]) + if mobj is None: + raise ValueError("no match for [%s]" % (flds[0],)) + (hh, mm, ss, ms) = mobj.groups() + if ms: + c_time = datetime.time(int(hh), int(mm), int(ss), int(ms) * 1000) + else: + c_time = datetime.time(int(hh), int(mm), int(ss)) + cdt = datetime.datetime.combine(tick_date, c_time) + x.append(date2num(cdt)) + y.append(last) + finally: + fh.close() + # throw away first line of file (close price from previous day) + del x[0] + del y[0] + return (x, y) + + +def main(): + LOG.debug("Loading ticks...") + x, y = tdl(datetime.datetime(2009,6,3)) + LOG.debug("Ticks loaded.") + + fig = plt.figure() + ax1 = fig.add_subplot(311) # ticks + # ax2 = fig.add_subplot(312) # gearing + # ax3 = fig.add_subplot(313) # cash + + ax1.set_ylabel("ticks") + # ax2.set_ylabel("gearing") + # ax3.set_ylabel("cash") + + xr = [x[0]] * 500 + yr = [y[0]] * 500 + + line, = ax1.plot_date(xr, yr, '-', xdate = True) + major_fmt = mdates.DateFormatter('%H:%M:%S') + ax1.xaxis.set_major_formatter(major_fmt) + # ax1.axis('tight') + + # format the coords message box + def price(x): return '%1.2f'%x + # ax.format_xdata = mdates.DateFormatter('%Y-%m-%d') + ax1.format_xdata = mdates.DateFormatter('%H:%M:%S') + ax1.format_ydata = price + ax1.grid(True) + + # rotates and right aligns the x labels, and moves the bottom of the + # axes up to make room for them + fig.autofmt_xdate() + + def animate(): + w = 1000 + bias = 10 + + ymin = min(y[0:100]) + ymax = max(y[0:100]) + + low = ymin - bias + high = ymax + bias + trend = 0 + + for i in range(0, len(x)-w): + # prepare timeline window + xr = x[i:i+w] + yr = y[i:i+w] + # update line + line.set_xdata(xr) + line.set_ydata(yr) + # determine y axis + if yr[-1] > ymax: + ymax = yr[-1] + if ymax - 50 < min(yr): + ymin = ymax - 50 + if yr[-1] < ymin: + ymin = yr[-1] + if ymin + 50 > max(yr): + ymax = ymin + 50 + ax1.axis([xr[0], xr[-1], ymin, ymax]) + # check low high and annotate + if i > 0 and i % 150 == 0: + ax1.annotate('low/high', + xy = (xr[-10], yr[-10]), + xytext = (xr[-5], yr[-10] + 10), + arrowprops = dict(facecolor = 'black', + frac = 0.2, + headwidth = 10, + linewidth = 0.1, + shrink = 0.05)) + + fig.canvas.draw() + + root = fig.canvas.manager.window + root.after(100, animate) + plt.show() + + +if __name__ == "__main__": + main() diff --git a/mpl/mpl-embedded.py b/mpl/mpl-embedded.py new file mode 100644 index 0000000..2d5de0f --- /dev/null +++ b/mpl/mpl-embedded.py @@ -0,0 +1,330 @@ +# Copyright (c) 2008 Andreas Balogh +# See LICENSE for details. + +""" animated drawing of ticks + + using self.canvas embedded in Tk application + + Fibionacci retracements of 61.8, 50.0, 38.2, 23.6 % of min and max +""" + +# system imports + +import datetime +import os +import re +import logging +import sys +import warnings + +import Tkinter as Tk +import numpy as np + +import matplotlib +matplotlib.use('TkAgg') + +from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2TkAgg +from matplotlib.figure import Figure +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +from matplotlib.dates import date2num + +# local imports + +# constants + +# globals + +LOG = logging.getLogger() + +logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s.%(msecs)03i %(levelname).4s %(process)d:%(thread)d %(message)s', + datefmt='%H:%M:%S') + +MDF_REO = re.compile("(..):(..):(..)\.*(\d+)*") + +def tdl(tick_date): + """ returns a list of tick tuples (cdt, last) for specified day """ + fiid = "846900" + year = tick_date.strftime("%Y") + yyyymmdd = tick_date.strftime("%Y%m%d") + filename = "%s.csv" % (fiid) + filepath = os.path.join("d:\\rttrd-prd-var\\consors-mdf\\data", year, yyyymmdd, filename) + x = [ ] + y = [ ] + fh = open(filepath, "r") + try: + prev_last = "" + for line in fh: + flds = line.split(",") + # determine file version + if flds[2] == "LAST": + last = float(flds[3]) + vol = float(flds[4]) + else: + last = float(flds[4]) + vol = 0.0 + # skip ticks with same last price + if prev_last == last: + continue + else: + prev_last = last + # parse time + mobj = MDF_REO.match(flds[0]) + if mobj is None: + raise ValueError("no match for [%s]" % (flds[0],)) + (hh, mm, ss, ms) = mobj.groups() + if ms: + c_time = datetime.time(int(hh), int(mm), int(ss), int(ms) * 1000) + else: + c_time = datetime.time(int(hh), int(mm), int(ss)) + cdt = datetime.datetime.combine(tick_date, c_time) + x.append(date2num(cdt)) + y.append(last) + finally: + fh.close() + # throw away first line of file (close price from previous day) + del x[0] + del y[0] + return (x, y) + + +class Mmh: + """Time series max & min detector.""" + def __init__(self, bias): + assert(bias > 0) + self.bias = bias + self.trend = None + self.mm0 = None + self.mms = [ ] + self.mins = [ ] + self.maxs = [ ] + + def __call__(self, tick): + """Add extended tick to the max min parser. + + @param tick: The value of the current tick. + @type tick: tuple(cdt, last) + + @return: 1. Tick if new max min has been detected, + 2. None otherwise. + """ + n, cdt, last = tick + res = None + # automatic initialisation + if self.mm0 is None: + # initalise water mark + self.mm0 = tick + res = self.mm0 + self.mins = [(n, cdt, last - 1)] + self.maxs = [(n, cdt, last + 1)] + else: + # initalise trend until price has changed + if self.trend is None or self.trend == 0: + self.trend = cmp(last, self.mm0[2]) + # check for max + if self.trend > 0: + if last > self.mm0[2]: + self.mm0 = tick + if last < self.mm0[2] - self.bias: + self.mms.insert(0, self.mm0) + self.maxs.append(self.mm0) + res = self.mm0 + # revert trend & water mark + self.mm0 = tick + self.trend = -1 + # check for min + if self.trend < 0: + if last < self.mm0[2]: + self.mm0 = tick + if last > self.mm0[2] + self.bias: + self.mms.insert(0, self.mm0) + self.mins.append(self.mm0) + res = self.mm0 + # revert trend & water mark + self.mm0 = tick + self.trend = +1 + return res + + +class Main: + def __init__(self): + LOG.debug("Loading ticks...") + self.x, self.y = tdl(datetime.datetime(2009,6,3)) + LOG.debug("Ticks loaded.") + self.mmh = Mmh(10) + + self.root = Tk.Tk() + self.root.wm_title("Embedding in TK") + + fig = plt.figure() + self.ax1 = fig.add_subplot(111) # ticks + # ax2 = fig.add_subplot(312) # gearing + # ax3 = fig.add_subplot(313) # cash + + self.ax1.set_ylabel("ticks") + # ax2.set_ylabel("gearing") + # ax3.set_ylabel("cash") + + self.w = 1000 + self.bias = 10 + + xr = self.x[0:self.w] + yr = self.y[0:self.w] + + fit = np.average(yr) + + self.tl, = self.ax1.plot_date(xr, yr, '-') + self.fl, = self.ax1.plot_date(xr, (fit, ) * self.w, 'k--') + self.mh, = self.ax1.plot_date(xr, (yr[0], ) * self.w, 'g:') + self.ml, = self.ax1.plot_date(xr, (yr[0], ) * self.w, 'r:') + major_fmt = mdates.DateFormatter('%H:%M:%S') + self.ax1.xaxis.set_major_formatter(major_fmt) + + self.ax1.format_xdata = mdates.DateFormatter('%H:%M:%S') + self.ax1.format_ydata = lambda x: '%1.2f'%x + self.ax1.grid(True) + + # rotates and right aligns the x labels, and moves the bottom of the + # axes up to make room for them + fig.autofmt_xdate() + self.ax1.axis([xr[0], xr[-1]+180./86400., min(yr), max(yr)]) + + self.canvas = FigureCanvasTkAgg(fig, master=self.root) + self.canvas.draw() + self.canvas.get_tk_widget().pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) + + # toolbar = NavigationToolbar2TkAgg( self.canvas, self.root ) + # toolbar.update() + # self.canvas._tkself.canvas.pack(side=Tk.TOP, fill=Tk.BOTH, expand=1) + + fr1 = Tk.Frame(master = self.root) + bu1 = Tk.Button(master = fr1, text='Quit', command=self.root.quit) + bu2 = Tk.Button(master = fr1, text='Stop', command=self.stop) + bu3 = Tk.Button(master = fr1, text='Resume', command=self.resume) + bu1.pack(side = Tk.RIGHT, padx = 5, pady = 5) + bu2.pack(side = Tk.RIGHT, padx = 5, pady = 5) + bu3.pack(side = Tk.RIGHT, padx = 5, pady = 5) + fr1.pack(side = Tk.BOTTOM) + + def animate_start(self): + warnings.simplefilter("default", np.RankWarning) + + self.ymin = min(self.y[0:self.w]) + self.ymax = max(self.y[0:self.w]) + + self.low = self.ymin - self.bias + self.high = self.ymax + self.bias + self.trend = 0 + self.i = 0 + for n in range(0, self.w): + self.mark_low_high(n) + self.root.after(500, self.animate_step) + + def animate_step(self): + # prepare timeline window + xr = np.array( self.x[self.i:self.i+self.w] ) + yr = np.array( self.y[self.i:self.i+self.w] ) + # update line + self.tl.set_xdata(xr) + self.tl.set_ydata(yr) + # determine y axis + if yr[-1] > self.ymax: + self.ymax = yr[-1] + if self.ymax - 50 < min(yr): + self.ymin = self.ymax - 50 + if yr[-1] < self.ymin: + self.ymin = yr[-1] + if self.ymin + 50 > max(yr): + self.ymax = self.ymin + 50 + # check self.low self.high and annotate + fwd = 2 + for n in range(self.i+self.w, self.i+self.w+fwd): + self.mark_low_high(n) + self.i += fwd + # build polynomial fit + mms = self.mmh.mms + if len(mms) > 1: + # mx = [ (x - int(x)) * 86400 for n, x, y in mms[:4] ] + # my = [ y for n, x, y in mms[:4] ] + mx = [ (x - int(x)) * 86400 for x in xr ] + my = yr + xre = [ xr[-1] + x/86400. for x in range(1, 181)] + xr2 = np.append(xr, xre) + # print "mx: ", mx + # print "my: ", my + polyval = np.polyfit(mx, my, 30) + # print "poly1d: ", polyval + intx = np.array(xr, dtype = int) + sodx = xr - intx + sodx *= 86400 + s2x = np.append(sodx, range(sodx[-1], sodx[-1]+180)) + fit = np.polyval(polyval, s2x) + self.fl.set_xdata(xr2) + self.fl.set_ydata(fit) + maxs = self.mmh.maxs + if len(maxs) > 1: + n, x1, y1 = maxs[-2] + n, x2, y2 = maxs[-1] + x3 = xr[-1] + polyfit = np.polyfit((x1, x2), (y1, y2), 1) + y3 = np.polyval(polyfit, x3) + self.mh.set_data((x1, x2, x3), (y1, y2, y3)) + mins = self.mmh.mins + if len(mins) > 1: + n, x1, y1 = mins[-2] + n, x2, y2 = mins[-1] + x3 = xr[-1] + polyfit = np.polyfit((x1, x2), (y1, y2), 1) + y3 = np.polyval(polyfit, x3) + self.ml.set_data((x1, x2, x3), (y1, y2, y3)) + # draw + self.ax1.axis([xr[0], xr[-1]+180./86400., self.ymin, self.ymax]) + self.canvas.draw() + if self.i < len(self.x)-self.w-1: + self.after_id = self.root.after(10, self.animate_step) + + def build_fit(self): + pass + + def mark_low_high(self, n): + x = self.x + y = self.y + rc = self.mmh((n, x[n], y[n])) + if rc: + nlh, xlh, ylh = rc + if self.mmh.trend > 0: + # low + self.ax1.annotate('low', + xy = (x[nlh], y[nlh]), + xytext = (x[n], y[nlh]), + arrowprops = dict(facecolor = 'red', + shrink = 0.05)) + # an.set_annotation_clip(True) + elif self.mmh.trend < 0: + # high + self.ax1.annotate('high', + xy = (x[nlh], y[nlh]), + xytext = (x[n], y[nlh]), + arrowprops = dict(facecolor = 'green', + shrink = 0.05)) + # an.set_annotation_clip(True) + + def stop(self): + if self.after_id: + self.root.after_cancel(self.after_id) + self.after_id = None + + def resume(self): + if self.after_id is None: + self.after_id = self.root.after(10, self.animate_step) + + def run(self): + self.root.after(500, self.animate_start) + self.root.mainloop() + self.root.destroy() + + +if __name__ == "__main__": + app = Main() + app.run() diff --git a/src/batch_rename.py b/src/batch_rename.py new file mode 100644 index 0000000..45ad256 --- /dev/null +++ b/src/batch_rename.py @@ -0,0 +1,62 @@ +""" flatex broker server """ + +import getopt +import sys +import os +import re + +import logging +mylog = logging.getLogger('batch_rename') + +class Usage(Exception): + def __init__(self, message): + Exception.__init__(message) + + +class Error(Exception): + def __init__(self, message): + Exception.__init__(message) + +# Star.Trek.The.Next.Generation.1x01-02.Encounter.At.Farpoint.DVDRip-AMC.[tvu.org.ru].avi +# Star Trek TNG - 1x01-2 - Encounter At Farpoint - DVDRip-AMC.divx.avi + +def main(argv = None): + if argv == None: + argv = sys.argv + mylog.info("starting [%s]" % (argv[0],)) + try: + try: + opts, args = getopt.getopt(argv[1:], "h", ["help"]) + print "opts: ", opts, "args:", args + except getopt.error, msg: + raise Usage(msg) + # more code, unchanged + p = re.compile("Star\.Trek\.The\.Next\.Generation.(.*?)\.(.*)\.(DVDRip-.*)\.\[tvu.org.ru\]\.avi") + tng = "e:\\share\\movies\\star trek tng" + os.chdir(tng) + for (dirpath, dirnames, filenames) in os.walk(tng): + for f in filenames: + mot = re.match(p, f) + if mot is not None: + # print mot.group() + print mot.groups() + ver = mot.group(1) + title0 = mot.group(2) + title1 = title0.replace('.', ' ') + nf = "Star Trek TNG - %s - %s - %s.divx.avi" % (ver, title1, mot.group(3)) + # os.rename(f, nf) + print 'ren "%s" "%s"' % (f, nf) + + except Error, err: + mylog.error(err.message) + return 1 + except Usage, err: + mylog.error(err.message) + mylog.error("for help use --help") + return 2 + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s.%(msecs)03d %(levelname)-8s %(message)s', + datefmt='%H:%M:%S') + sys.exit(main()) diff --git a/src/ftpget.py b/src/ftpget.py new file mode 100644 index 0000000..b88cb4e --- /dev/null +++ b/src/ftpget.py @@ -0,0 +1,6 @@ +from ftplib import FTP + +ftp = FTP('ftp.asstr.org') # connect to host, default port +ftp.login() # user anonymous, passwd anonymous@ +print ftp.getwelome() +ftp.retrbinary('RETR README', open('README', 'wb').write) \ No newline at end of file diff --git a/src/histogram.py b/src/histogram.py new file mode 100644 index 0000000..e433b74 --- /dev/null +++ b/src/histogram.py @@ -0,0 +1,97 @@ +""" walk histogram +""" + +import sys +import re +import datetime + +mol = re.compile("\S+\s(\S+?)\|.{5}(.{6}).\|(.*)\|(.*)") + +def main(argv = None): + if argv == None: + argv = sys.argv + ifn = "e:/livefeed/20070829/GDAXI.csv" + ofn = "e:/eclipse/workspace/filter/var/GDAXI_histogram.csv" + tick_list = [ ] + histogram = { } + print "Converting [%s] to [%s]" % (ifn, ofn) + try: + ifh = open(ifn, "r") + ofh = open(ofn, "w") + except IOError: + print "open [%s] failed" % ifn + return 2 + try: + for line in ifh: + tick_list.append(parseCsv(line)) + for i in xrange(1, len(tick_list)): + diff = tick_list[i].last - tick_list[i-1].last + key = int( diff * 100 ) + if key not in histogram: + histogram[key] = 1 + else: + histogram[key] += 1 + # for key in sorted(histogram.keys()): + for i in xrange(-1000, 1000): + if i in histogram: + print >> ofh, float(i)/100, ",", histogram[i] + else: + print >> ofh, float(i)/100, ",0" + finally: + ifh.close() + ofh.close() + print "Done." + +time_pattern = re.compile("(..):(..):(..).(\d+)") + +def parseCsv(line): + flds = line.split(',') + mot = re.match(time_pattern, flds[0]) + (hh, mm, ss, ms) = mot.group(1, 2, 3, 4) + time = datetime.time(int(hh), int(mm), int(ss), int(ms)*1000) + # pid = flds[1] + bid = flds[2] + ask = flds[3] + last = flds[4] + return Tick(time, float(bid), float(ask), float(last)) + +class Tick: + """ VDO helper class """ + def __init__(self, ti, bid, ask, last): + self.time = ti + self.bid = bid + self.ask = ask + self.last= last + + def __cmp__(self, other): + return cmp(self.time, other.time) + +class Roi: + """ VDO helper class """ + def __init__(self, buy_time, sell_time, pnl): + self.buy_time = buy_time + self.sell_time = sell_time + self.pnl = pnl + + def pnl_per_minute(self): + dt = time_in_minutes(self.sell_time) - time_in_minutes(self.buy_time) + try: + ppm = self.pnl / dt + except ZeroDivisionError: + return self.pnl + return ppm + + def __cmp__(self, other): + return cmp(self.pnl_per_minute(), other.pnl_per_minute()) + + def __repr__(self): + return "%s,%s,%.2f,%.2f" % (self.buy_time, self.sell_time, + self.pnl, self.pnl_per_minute()) + + +def time_in_minutes(ti): + return ti.hour * 60 + ti.minute + float(ti.second) / 60 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/src/pricefeed.py b/src/pricefeed.py new file mode 100644 index 0000000..cb4df2d --- /dev/null +++ b/src/pricefeed.py @@ -0,0 +1,35 @@ +""" pricefeed file filter +*.log: "20070905 09:04:56.841|DE000DB0M091|4.430|4.440" +*.csv: "09:04:56.841,DB0M09,4.430,4.440" +""" + +import sys +import re + +mol = re.compile("\S+\s(\S+?)\|.{5}(.{6}).\|(.*)\|(.*)") + +def main(argv = None): + if argv == None: + argv = sys.argv + if (len(argv) != 2): + print "Usage: filter.py " + return 2 + ifn = argv[1] + ofn = "".join((ifn.split(".")[0], ".csv")) + print "Converting [%s] to [%s]" % (ifn, ofn) + try: + ifh = open(ifn, "r") + ofh = open(ofn, "w") + except IOError: + print "open [%s] failed" % ifn + return 2 + try: + for line in ifh: + flds = mol.match(line) + print >>ofh, ",".join(flds.groups()) + finally: + ifh.close() + ofh.close() + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/src/roimatrix.py b/src/roimatrix.py new file mode 100644 index 0000000..a30f73f --- /dev/null +++ b/src/roimatrix.py @@ -0,0 +1,94 @@ +""" roi matrix +""" + +import sys +import re +import datetime + +mol = re.compile("\S+\s(\S+?)\|.{5}(.{6}).\|(.*)\|(.*)") + +def main(argv = None): + if argv == None: + argv = sys.argv + ifn = "e:/livefeed/20070829/GDAXI.csv" + ofn = "e:/eclipse/workspace/filter/var/GDAXI_roi.csv" + tick_list = [ ] + roi_list = [ ] + print "Converting [%s] to [%s]" % (ifn, ofn) + try: + ifh = open(ifn, "r") + ofh = open(ofn, "w") + except IOError: + print "open [%s] failed" % ifn + return 2 + try: + for line in ifh: + tick_list.append(parseCsv(line)) + for (i, buy_tick) in enumerate(tick_list[0::300]): + print buy_tick.time + for sell_tick in tick_list[0::300]: + if (buy_tick > sell_tick): + continue + pnl = ( sell_tick.last - buy_tick.last ) * 25.0 - 11.80 + # print buy_tick.last, sell_tick.last, pnl + roi_list.append(Roi(buy_tick.time, sell_tick.time, pnl)) + for roi in roi_list: + print >> ofh, str(roi) + finally: + ifh.close() + ofh.close() + print "Done." + +time_pattern = re.compile("(..):(..):(..).(\d+)") + +def parseCsv(line): + flds = line.split(',') + mot = re.match(time_pattern, flds[0]) + (hh, mm, ss, ms) = mot.group(1, 2, 3, 4) + time = datetime.time(int(hh), int(mm), int(ss), int(ms)*1000) + # pid = flds[1] + bid = flds[2] + ask = flds[3] + last = flds[4] + return Tick(time, float(bid), float(ask), float(last)) + +class Tick: + """ VDO helper class """ + def __init__(self, ti, bid, ask, last): + self.time = ti + self.bid = bid + self.ask = ask + self.last= last + + def __cmp__(self, other): + return cmp(self.time, other.time) + +class Roi: + """ VDO helper class """ + def __init__(self, buy_time, sell_time, pnl): + self.buy_time = buy_time + self.sell_time = sell_time + self.pnl = pnl + + def pnl_per_minute(self): + dt = time_in_minutes(self.sell_time) - time_in_minutes(self.buy_time) + try: + ppm = self.pnl / dt + except ZeroDivisionError: + return self.pnl + return ppm + + def __cmp__(self, other): + return cmp(self.pnl_per_minute(), other.pnl_per_minute()) + + def __repr__(self): + return "%s,%s,%.2f,%.2f" % (self.buy_time, self.sell_time, + self.pnl, self.pnl_per_minute()) + + +def time_in_minutes(ti): + return ti.hour * 60 + ti.minute + float(ti.second) / 60 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/src/wget_batch.py b/src/wget_batch.py new file mode 100644 index 0000000..c365ebc --- /dev/null +++ b/src/wget_batch.py @@ -0,0 +1,52 @@ +""" broker quote request recorder engine """ + +import logging +import getopt +import sys +import datetime + +mylog = logging.getLogger() + + +class Usage(Exception): + def __init__(self, message): + Exception.__init__(message) + + +class Error(Exception): + def __init__(self, message): + Exception.__init__(message) + + +def main(argv = None): + if argv == None: + argv = sys.argv + mylog.info("starting [%s]" % (argv[0],)) + try: + try: + opts, args = getopt.getopt(argv[1:], "h", ["help"]) + print "opts: ", opts, "args:", args + except getopt.error, msg: + raise Usage(msg) + # set up stderr logging + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s.%(msecs)03d %(levelname)-8s %(message)s', + datefmt='%H:%M:%S') + # main code starts here + ofh = open("e:/install/asstr_get", "w") + for i in xrange(56546,57096): + # print >> ofh, "wget http://www.literotica.com/stories/showstory.php?id=%i" % i + print >> ofh, "wget http://www.asstr.org/files/Collections/Alt.Sex.Stories.Moderated/Year2007/%i" % i + ofh.close() + + except Error, err: + mylog.error(err.message) + return 1 + except Usage, err: + mylog.error(err.message) + mylog.error("for help use --help") + return 2 + +if __name__ == '__main__': + sys.exit(main()) +