"""
This is an experimental module.
Everything in this module is untested and probably incorrect.
Don't use them.

This is intended to be a place to develop new functions instead of
having an entirely new branch
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from numba import jit, njit
import os
import inspect
from intake import Schema

class DataSource:
    """
    A dummy datasource class replacing the intake datasource
    class to enable fast loading of this module
    This is a temporary ugly fix
    Intake catalog classes must be moved to a separate module
    """

    def __init__(self):
        pass

@jit
def v_cusum(array):
    """
    Calcuate cusum - numba version
    array
        numpy array
    returns
        pos and neg arrays
    """
    len(array)
    pos = [0]
    neg = [0]
    pos_val = 0
    neg_val = 0
    d = np.diff(array)[1:]
    for i in d:
        if i >= 0:
            pos_val += i
        else:
            neg_val += i
        pos.append(pos_val)
        neg.append(neg_val)
    return (pos, neg)

@jit
def sign_change(array):
    """
    Calcuate the sign change in an array
    If the current value is positive and previous value negative, mark as 1.
    If the current value is negative and previous value positive, mark as -1.
    In case of no change in sign, mark as 0
    """
    L = len(array)
    arr = np.empty(L)
    arr[0] = 0
    for i in range(1, L):
        # TO DO: Condition not handling edge case
        if (array[i] >= 0) & (array[i - 1] < 0):
            arr[i] = 1
        elif (array[i] <= 0) & (array[i - 1] > 0):
            arr[i] = -1
        else:
            arr[i] = 0
    return arr

def cusum(array):
    """
    Calcuate cusum
    array
        a pandas series with a timestamp or datetime index
    The cusum is just an aggregate of positive and negative differences
    returns
        pandas dataframe with positive and negative cumulatives,
        ratio, differences, regime change along with the original index
    """
    pos = [0]
    neg = [0]
    pos_val = 0
    neg_val = 0
    d = array.diff()[1:]
    for i in d:
        if i >= 0:
            pos_val += i
        else:
            neg_val += i
        pos.append(pos_val)
        neg.append(neg_val)
    df = pd.DataFrame({"pos": pos, "neg": neg}, index=array.index)
    df["neg"] = df["neg"].abs()
    df["d"] = df["pos"] - df["neg"]
    df["reg"] = sign_change(df.d.values)
    df["ratio"] = df["pos"] / df["neg"]
    return df

def percentage_bar(data, step):
    """
    Generate the number of timesteps taken for each
    equivalent step in price
    data
        numpy 1d array
    step
        step size
    """
    start = data[0]
    nextStep = start + step
    counter = 0
    steps = [start]
    period = [0]
    for d in data:
        if step >= 0:
            if d > nextStep:
                steps.append(nextStep)
                period.append(counter)
                nextStep += step
                counter = 0
            else:
                counter += 1
        elif step < 0:
            if d < nextStep:
                steps.append(nextStep)
                period.append(counter)
                nextStep += step
                counter = 0
            else:
                counter += 1

    # Final loop exit
    steps.append(nextStep)
    period.append(counter)
    return (steps, period)

def high_breach(s):
    """
    Given a series of values, returns a series
    with consecutive highs as values and timestamp as index
    s
        series with timestamp as index
    """
    highs = []
    ts = []
    max_val = 0
    index = s.index.values
    for i, v in enumerate(s.values):
        if v > max_val:
            highs.append(v)
            ts.append(index[i])
            max_val = v
    return pd.Series(highs, index=ts)

def low_breach(s):
    """
    Given a series of values, returns a series
    with consecutive lows as values and timestamp as index
    s
        series with timestamp as index
    """
    lows = []
    ts = []
    min_val = 1e9  # Just setting an extreme value
    index = s.index.values
    for i, v in enumerate(s.values):
        if v < min_val:
            lows.append(v)
            ts.append(index[i])
            min_val = v
    return pd.Series(lows, index=ts)

class ExcelSource(DataSource):
    container = "dataframe"
    name = "excel_loader"
    version = "0.0.1"
    partition_access = True

    def __init__(self, datapath, metadata=None):
        """
        Initialize with datapath and metadata
        datapath
            filename with entire path
        """
        self.filename = datapath
        self._source = pd.ExcelFile(self.filename)
        super(ExcelSource, self).__init__(metadata=metadata)

    def _get_schema(self):
        sheets = self._source.sheet_names
        return Schema(
            datashape=None,
            dtype=None,
            shape=None,
            npartitions=len(sheets),
            extra_metadata={"sheets": sheets},
        )

    def read_partition(self, sheet, **kwargs):
        """
        Read a specific sheet from the list of sheets
        sheet
            sheet to read
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        if sheet in self.metadata.get("sheets", []):
            return self._source.parse(sheet, **kwargs)
        else:
            return "No such sheet in the Excel File"

    def read(self, **kwargs):
        """
        Read all sheets into a single dataframe.
        Sheetname is added as a column
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        sheets = self.metadata.get("sheets")
        collect = []
        if len(sheets) > 1:
            for sheet in sheets:
                temp = self.read_partition(sheet, **kwargs)
                temp["sheetname"] = sheet
                collect.append(temp)
        return pd.concat(collect, sort=False)

    def _close(self):
        self._source.close()

class HDFSource(DataSource):
    """
    A simple intake container to load data from
    HDF5 fixed formats
    """

    container = "dataframe"
    name = "HDF5_fixed_loader"
    version = "0.0.1"
    partition_access = True

    def __init__(self, datapath, metadata=None, extension="h5"):
        """
        Initialize with datapath and metadata
        datapath
            filename or directory
            If the filename ends in any of the extensions given in the
            extension argument, then it is treated as a HDF5 file
        """
        self.source = datapath
        self._ext = extension
        # Check whether the given path is a directory or file
        if os.path.exists(datapath):
            if os.path.isfile(datapath):
                self._source_type = "file"
            else:
                self._source_type = "directory"
        else:
            print("Not a valid file or directory")
            return
        self._get_schema()
        super(HDFSource, self).__init__(metadata=metadata)

    def _get_schema(self):
        metadata = {"ext": self._ext, "src": self.source, "type": self._source_type}
        file_dict = {}
        if self._source_type == "directory":
            for root, directory, files in os.walk(self.source):
                for file in files:
                    filename = os.path.join(root, file)
                    if filename.endswith(self._ext):
                        file_dict[file] = filename
            metadata.update({"files": file_dict})

        return Schema(
            datashape=None,
            dtype=None,
            shape=None,
            npartitions=len(file_dict),
            extra_metadata=metadata,
        )

    def read_partition(self, file, **kwargs):
        """
        Read a specific sheet from the list of sheets
        file
            filename without extension
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        ext = self.metadata.get("ext", self._ext)
        srctype = self.metadata.get("type")
        if srctype == "file":
            return pd.read_hdf(self.metadata.get("src"))
        filename = "{file}.{ext}".format(file=file, ext=ext)
        if filename in self.metadata.get("files", []):
            filepath = self.metadata["files"][filename]
            return pd.read_hdf(filepath)
        else:
            return "No such HDF file"

    def read(self, **kwargs):
        """
        Read all sheets into a single dataframe.
        Sheetname is added as a column
        kwargs
            kwargs to the excel parse function
        """
        self._load_metadata()
        if self.metadata.get("type") == "file":
            return pd.read_hdf(self.metadata.get("src"))
        else:
            return "The datasource is a directory.Use the read_partition method to read a specific file."

    def _close(self):
        print("Not implemented")

def twin_plot(data, y_axis, x_axis="timestamp"):
    """
    Create a bokeh plot with twin axes
    """
    from bokeh.plotting import figure
    from bokeh.models import LinearAxis, Range1d

    TOOLTIPS = [("datetime", "@x{%F %H:%M}"), ("value", "$y{0.00}")]

    y1, y2 = y_axis[0], y_axis[1]
    h0 = data[y1].max()
    l0 = data[y1].min()
    h1 = data[y2].max()
    l1 = data[y2].min()
    p = figure(
        x_axis_type="datetime",
        y_range=(l0, h0),
        tooltips=TOOLTIPS,
        height=240,
        width=600,
    )
    p.line(data[x_axis].values, data[y1].values, color="red", legend=y1)
    p.extra_y_ranges = {"foo": Range1d(l1, h1)}
    p.line(data[x_axis], data[y2].values, color="blue", y_range_name="foo", legend=y2)
    p.add_layout(LinearAxis(y_range_name="foo", axis_label=y2), "left")
    p.hover.formatters = {"x": "datetime"}
    p.legend.location = "top_center"
    p.legend.click_policy = "hide"
    return p

def conditional(data, c1, c2, out=None):
    """
    Create a conditional probability table with counts
    data
        dataframe
    c1
        condition as string
    c2
        list of conditions as strings
    out
        output format. If None, counts are returned.
        If a function is passed, it is applied to
        each of the conditions and the return value
        of the function is stored for each condition.
        The function should have a single argument that
        takes a dataframe as an input.

    returns a dictionary with the conditions and
    the counts or the return value of each of the conditions
    Note
    ----
    1. The dataframe is queried with c1 and each of the conditions
    in c2 are evaluated based on this result.
    2. All conditions are evaluated using `df.query`
    3. The condition strings should be valid columns in the dataframe
    4. The function passed should have a single argument, the dataframe.
    """
    dct = {}
    if out is None:
        out = len
    df = data.query(c1)
    dct[c1] = out(df)
    for c in c2:
        dct[c] = out(df.query(c))
    return dct

class Catalog:
    """
    A intake catalog creator
    The catalog is created in the following manner
        1. All files in the root directory are considered to
        be separate files and loaded as such.
        2. All directories and subdirectories inside the root
        directory are considered to be separate data sources
        3. Each of the files are matched against the extension
        name and the corresponding mapper
        4. Files inside a directory are randomly selected and the
        file type is determined for the entire directory.
        **It's assumed that all files inside any sub-directories
        are of the same file type**
    """

    def __init__(self, directory):
        """
        directory
            directory to search for files
        """
        self._directory = directory
        """
        All files in the below directories are added
        individually as a data source
        """
        self._file_dirs = ["files"]
        """
        **filetypes** is a dictionary with the file type as
        key and a sub-dictionary with driver and extensions
        as keys and the corresponding driver and extensions
        as values.
        It is a logical structure that maps a file type to
        its intake driver since each file type can have more
        than one extension. This dictionary is looped to get
        the self._mappers for each extension.
        **Assumed each filetype has a single driver but more than
        one extension**
        """
        filetypes = {
            "excel": {
                "driver": "fastbt.experimental.ExcelSource",
                "extensions": ["xls", "xlsx"],
            },
            "csv": {
                "driver": "intake.source.csv.CSVSource",
                "extensions": ["csv", "txt"],
            },
            "hdf": {
                "driver": "fastbt.experimental.HDFSource",
                "extensions": ["h5", "hdf5"],
            },
        }
        self._mappers = {}
        for k, v in filetypes.items():
            for ext in v["extensions"]:
                self._mappers[ext] = v["driver"]

    def generate_catalog(self):
        """
        Generate catalog
        #TO DO#
        1. Replace multiple dots with underscores in filenames
        """
        dct = {}
        dct["sources"] = {}
        src = dct["sources"]

        def metadata():
            """
            metadata generation for the file; has access to
            all variables inside the parent function
            """
            return {
                "args": {first_arg: os.path.join(dirpath, file)},
                "driver": self._mappers[ext],
                "description": "",
                "metadata": {"extension": ext, "mode": mode},
            }

        for dirpath, dirnames, filenames in os.walk(self._directory):
            dirname = dirpath.split("/")[-1]  #
            if dirname in self._file_dirs:
                mode = "file"
                for file in filenames:
                    ext = file.split(".")[-1]
                    if "csv" in ext:
                        first_arg = "urlpath"
                    else:
                        first_arg = "datapath"
                    if ext in self._mappers:
                        src[file.split(".")[0]] = metadata()
            else:
                mode = "dir"
                # If the directory has any files
                if len(filenames) > 0:
                    # Check the extension of the first file in directory
                    ext = filenames[0].split(".")[-1]
                    file = "*"
                    if "csv" in ext:
                        first_arg = "urlpath"
                        file = "*"
                    else:
                        first_arg = "datapath"
                        file = ""
                    if ext in self._mappers:
                        src[dirname] = metadata()
        return dct

def candlestick_plot(data):
    """
    return a bokeh candlestick plot
    data
        dataframe with open,high,low and close columns
    Note
    -----
    Prototype copied from the below link
    https://bokeh.pydata.org/en/latest/docs/gallery/candlestick.html
    """
    from math import pi
    from bokeh.plotting import figure
    from bokeh.models import ColumnDataSource

    df = data.copy()
    df["date"] = pd.to_datetime(df["date"])
    df["color"] = ["green" if x > y else "red" for (x, y) in zip(df.close, df.open)]
    source = ColumnDataSource()
    source.data = source.from_df(df)
    w = 10 * 60 * 1000  # half day in ms
    TOOLS = "pan,wheel_zoom,box_zoom,reset,save"
    p = figure(
        x_axis_type="datetime",
        tools=TOOLS,
        title="Candlestick",
        plot_width=800,
        tooltips=[
            ("date", "@date{%F}"),
            ("open", "@open{0}"),
            ("high", "@high{0}"),
            ("low", "@low{0}"),
            ("close", "@close{0}"),
        ],
    )
    p.hover.formatters = {"date": "datetime"}
    p.xaxis.major_label_orientation = pi / 4
    p.grid.grid_line_alpha = 0.3
    p.segment("date", "high", "date", "low", color="black", source=source)
    p.vbar(
        "date",
        w,
        "open",
        "close",
        fill_color="color",
        line_color="black",
        source=source,
    )
    return p, source

def calendar_plot(data, field="ret"):
    """
    return a calendar plot
    data
        dataframe with year and month columns
    field
        field to plot values
    Note
    -----
    Prototype copied from bokeh gallery
    https://bokeh.pydata.org/en/latest/docs/gallery/unemployment.html
    """
    from math import pi
    from bokeh.models import (
        LinearColorMapper,
        BasicTicker,
        PrintfTickFormatter,
        ColorBar,
    )
    from bokeh.plotting import figure
    from bokeh.palettes import Spectral

    df = data.copy()
    # Type conversion
    df["year"] = df.year.astype("str")
    df["month"] = df.month.astype("str")
    years = list(df.year.unique())
    months = [str(x) for x in range(12, 0, -1)]
    colors = list(reversed(Spectral[6]))
    mapper = LinearColorMapper(
        palette=colors, low=df[field].min(), high=df[field].max()
    )
    TOOLS = "hover,save,pan,box_zoom,reset,wheel_zoom"
    p = figure(
        title="Calendar Plot",
        x_range=years,
        y_range=list(reversed(months)),
        x_axis_location="above",
        plot_width=1000,
        plot_height=600,
        tools=TOOLS,
        toolbar_location="below",
        tooltips=[("date", "@year-@month"), ("return", "@{}".format(field))],
    )
    # Axis settings
    p.grid.grid_line_color = None
    p.axis.axis_line_color = None
    p.axis.major_tick_line_color = None
    p.axis.major_label_text_font_size = "10pt"
    p.axis.major_label_standoff = 0
    p.xaxis.major_label_orientation = pi / 3

    p.rect(
        x="year",
        y="month",
        width=1,
        height=1,
        source=df,
        fill_color={"field": field, "transform": mapper},
        line_color="black",
        line_width=0.5,
        line_alpha=0.3,
    )

    color_bar = ColorBar(
        color_mapper=mapper,
        major_label_text_font_size="8pt",
        ticker=BasicTicker(desired_num_ticks=len(colors)),
        formatter=PrintfTickFormatter(format="%d%%"),
        label_standoff=6,
        border_line_color=None,
        location=(0, 0),
    )
    p.add_layout(color_bar, "right")
    return p

def summary_plot(data):
    """
    Given a dataframe, create a widget with condition and list
    of columns to provide summary data
    data
        dataframe
    Note
    -----
     * Condition is a valid eval string supported by pandas
     * Multiple columns can be selected
     * Summary statistics provided by the describe method

    """
    from bokeh.models import (
        TextInput,
        Button,
        MultiSelect,
        PreText,
    )
    from bokeh.layouts import layout

    df = data.copy()

    def document(doc):
        condition = TextInput(title="Enter your condition")
        col = MultiSelect(title="Select the columns", options=list(df.columns))
        button = Button(label="Update")
        pre = PreText()
        pre.text = condition.value

        def update():
            cond = condition.value
            cols = col.value
            text = df.query(cond)[cols].describe()
            pre.text = str(text)

        button.on_click(update)

        l = layout([[[condition, button], col], pre])
        doc.add_root(l)

    return document

def slider_plot(data, cols):
    """
    Given a dataframe, create a widget with range slider
    data
        dataframe
    cols
        columns for which sliders are to be created
    """
    from bokeh.models import (
        Button,
        RangeSlider,
        MultiSelect,
        PreText,
    )
    from bokeh.layouts import column, layout

    df = data.copy()

    def document(doc):
        sliders = []
        pre = PreText(text="something")
        select = MultiSelect(title="Select the columns", options=list(df.columns))
        button = Button(label="Update")
        for col in cols:
            MIN, MAX = df[col].min(), df[col].max()
            STEP = (MAX - MIN) / 100
            slider = RangeSlider(
                start=MIN, end=MAX, step=STEP, value=(MIN, MAX), title=col
            )
            sliders.append(slider)

        def update():
            values = []
            txt = "({col} > {low}) & ({col} < {high})"
            for slider in sliders:
                low, high = slider.value
                low, high = float(low), float(high)
                formula = txt.format(col=slider.title, low=low, high=high)
                values.append(formula)
            q = "&".join(values)
            summary_cols = select.value
            text = df.query(q)[summary_cols].describe()
            pre.text = str(text)

        button.on_click(update)

        l = layout([column(sliders), [select, button]], pre)
        doc.add_root(l)

    return document

def run_simulation(data, size=0.3, num=1000, column=None, function=np.mean):
    """
    run a simulation on the given data by drawing repeated samples
    and evaluating a metric
    data
        a pandas dataframe
    size
        size of the sample - float/int
        if size is a float and less than , it is considered a percentage
        in case of a num it is considered as the number of samples to take
    num
        number of simulations to be run
    column
        column for which sample is to be taken.
        If None, the entire dataframe is sampled and this is considerably slow
    function
        function to be run on the column - string/function
        default mean

    returns a pandas Series with the result for each sample taken
    """
    choice = np.random.choice
    if size < 1:
        N = int(len(data) * size)
    else:
        N = size
    collect = []
    if column is None:
        # Take sample on the whole dataframe
        for i in range(num):
            sample = function(data.sample(N))
            collect.append(sample)
    else:
        # Take sample on a particular column
        # considerably faster since this is a numpy version
        col = data[column].values
        for i in range(num):
            sample = function(choice(col, N))
            collect.append(sample)
    return pd.Series(collect)

def generate_parameters(dict_of_parameters):
    """
    Generate a list of parameters for backtest function
    dict_of_parameters
        dictionary of parameters in the given format
    Note
    -----
    Only one level of nested dictionary is parsed
    """
    from itertools import product

    d = dict_of_parameters.copy()
    lst = []

    def simple(dictionary, updt):
        # An inner function for ugly lookup
        # TO DO: Think of better way -> recursion
        listed = []
        for k, v in dictionary.items():
            if isinstance(v, list):
                listed.append([{k: val} for val in v])
            elif isinstance(v, str):
                listed.append([{k: v}])
        temp = list(product(*listed))
        lstk = []
        for tp in temp:
            temp_dict = updt.copy()
            for t in tp:
                temp_dict.update(t)
            lstk.append(temp_dict)
        return lstk

    for k, v in d.items():
        if isinstance(v, list):
            lst.append([{k: val} for val in v])
        elif isinstance(v, str):
            lst.append([{k: v}])
        elif isinstance(v, dict):
            dict_lst = []
            for k1, v1 in v.items():
                lst2 = simple(d[k][k1], {k: k1})
                dict_lst.extend(lst2)
            lst.append(dict_lst)
    return lst

@njit
def traverse(high, low, points):
    """
    See whether the price points are hit in the given order
    high
        high values as numpy array
    low
        low values as numpy array
    points
        list or numpy array of values to check
    returns a 4-tuple with
      1. an array indicating whether the values in
      the points list is hit; 1 indicates the value is hit
      2. an array of timesteps indicating when the
      value is hit; this is just the iteration cycle
      3. high value when the last point is hit or the
      end of the iteration cycle if the point is not hit
      4. low value when the last point is hit or the
      end of the iteration cycle if the point is not hit
    Note
    ----
    1. This function checks whether the points are hit
    in the given order.
    2. A point is considered hit if it is between high and
    low value
    3. high and low values are considered to be array of equal
    size with the value of high always greater than or equal
    to the value of low at every timestep
    """
    j = 0
    price = points[j]
    hit = np.zeros(len(points))
    timesteps = np.zeros(len(points))
    for i in range(len(high)):
        if low[i] < price < high[i]:
            hit[j] = 1
            timesteps[j] = i
            j += 1
            if j == len(points):
                break
            else:
                price = points[j]
    return (hit, timesteps, high[i], low[i])

class Strategy:
    """
    An automated strategy implementation class
    """

    def __init__(self, **kwargs):
        """
        keyword arguments
        result_cols
            column names for any extra value emitted
            from the tradebook function. The first 4
            columns are automatically named entry_time,
            entry_price, exit_time and exit_price and
            must correspond accordingly
        """
        self.datas = []
        self.result_cols = kwargs.pop("result_cols", None)

    @staticmethod
    def tradebook(open, high, low, close, **kwargs):
        """
        Numpy tradebook implementation with logic.
        To be implemented for each case
        Note
        -----
        1) Should always return an array of values
        2) The array must be a one-dimensional array
        3) The first 4 values in the array must be
        entry time, entry price, exit time and exit price
        4) It is assumed only one trade is done each day
        5) For multiple trades, modify the result function
        """
        raise NotImplementedError

    def _agged(self, times, data=None):
        """
        Get the aggregated data for the given times
        data
            intraday dataframe. If None, self.datas[0] is picked
        times
            list of 2-tuples with start and end times
        Note
        -----
        1) For each time tuple given in list, data is aggregated
        in OHLC form and added to a dataframe
        2) Columns are numbered based on the order of the times
        list so that the first tuple is named 0, the next 1 and so on
        3) Expect a date and timestamp column in the list
        """
        from fastbt.utils import recursive_merge

        agged = {"open": "first", "high": "max", "low": "min", "close": "last"}
        dfs = []
        if not (data):
            data = self.datas[0]
        for i, (s, e) in enumerate(times):
            temp = (
                data.set_index("timestamp")
                .between_time(s, e)
                .groupby("date")
                .agg(agged)
            )
            columns = {
                x: "{x}{i}".format(x=x, i=i) for x in ["open", "high", "low", "close"]
            }
            temp.rename(columns=columns, inplace=True)
            dfs.append(temp)
        return recursive_merge(dfs, on=["date"]).reset_index()

    def _each_day(self, data=None, cols=None, **kwargs):
        """
        Runs the tradebook function on each day.
        data
            dataframe on which tradebook simulation is run.
            In case of None, the last item in self.datas is used
            The dataframe must have a date column and it must
            be sorted by timestamp
        cols
            columns to be passed to the tradebook function.
            The columns should be argument names in the
            tradebook function
        kwargs
            keyword arguments would be passed to the tradebook function
        """
        if not (data):
            data = self.datas[-1]  # the last appended data
        grouped = data.groupby("date")
        if not (cols):
            cols = ["open", "high", "low", "close"]
        tradebook = self.tradebook

        def tb(x):
            dct = {c: x[c].values for c in cols}
            kwargs.update(dct)
            return tradebook(**kwargs)

        res = grouped.apply(tb)
        return res

    def result(self, **kwargs):
        """
        Result of the strategy.
        The output of the tradebook function is the input to
        this function. Converts the array into a dataframe
        and adds some useful columns
        """
        tmp = self._each_day(**kwargs)

        def get_column_names():
            """
            Get column names
            """
            cols = ["entry_time", "entry_price", "exit_time", "exit_price"]
            if self.result_cols:
                print(cols + self.result_cols)
                return cols + self.result_cols
            else:
                L = len(tmp.iloc[0])
                return cols + [f"col{i}" for i in range(4, L)]

        res = pd.DataFrame(
            tmp.values.tolist(), index=tmp.index, columns=get_column_names()
        )
        res["year"] = res.index.year
        res["profit"] = res.eval("exit_price-entry_price")
        res["cum_p"] = res.profit.cumsum()
        res["max_p"] = res.cum_p.expanding().max()
        return res

    def metric(self):
        """
        Default metric to use
        """
        return

    def plot(self, data):
        """
        A general plotting function
        """
        data[["cum_p", "max_p"]].plot()
        plt.figure()
        data.groupby("year").profit.sum().plot.bar()

    def simulator(self, **kwargs):
        """
        Run simulation and return results
        kwargs
            keyword arguments where argument is the name of
            the variable and values as list the variable takes.
            The keyword variable is passed to the tradebook
            function apart from OHLC values
        """
        pass

    def run(self, plotting=True, **kwargs):
        """
        Run the entire process
        """
        res = self.result(**kwargs)
        if plotting:
            self.plot(res)
        return res  # For further analysis

def advances(data, date="date", column=None, out="advances"):
    """
    Get the advances by day
    data
        dataframe with the necessary columns
    date
        date column in the dataframe to group by
    column
        column on which ratio would be calculated, usually a returns column
        If column is not available, the close/open return is calculated
    out
        output format one of advances/declines/difference/ratio/all
        all returns everything
    """
    data = data.copy()
    if column is None:
        data["pret"] = data.eval("(close/open)-1")
        column = "pret"
    col = "is_" + column
    data[col] = data.eval(f"{column}>0") + 0
    data2 = data.groupby([date, col]).size().unstack().fillna(0).astype(int)
    data2.columns = ["declines", "advances"]
    data2["difference"] = data2.eval("advances-declines")
    data2["ratio"] = data2.eval("advances/declines")
    if out == "all":
        return data2
    else:
        return data2[out]

class CodeGenerator:
    def __init__(self, name, blocks=None):
        self._name = name
        self._struct = []
        self._blocks = {}
        self._block_names = []

    @property
    def name(self):
        return self._name

    def add_block(self, name, indent=False, n=4):
        """
        Add a code block
        name
            name of the code block
        indent
            whether to apply indentation to the code
            only one level of indentation is added
        Note
        -----
        1. name should be a key in the blocks dictionary
        2. indentation is provided with 4 spaces

        """
        self._block_names.append(name)
        if indent:
            txt = "{{" + str(name) + " | indent(" + str(n) + ", first=True)}}"
        else:
            txt = "{{" + str(name) + "}}"
        self._struct.append(txt)

    def add_text(self, text):
        """
        Add text
        text
            Text to add
        the given text is added without any variable interpolation
        """
        self._struct.append(text)

    def add_code_block(self, name, block):
        """
        Add a code block to the existing blocks dictionary
        name
            name of the block
        block
            actual code
        If name is already in the blocks dictionary,
        it would be overwritten
        """
        self._blocks[name] = block

    def clear(self):
        """
        Clear everything in the existing structure
        """
        self._struct = []
        self._block_names = []

    def generate_code(self):
        from jinja2 import Template

        code = "\n".join(self._struct)
        template = Template(code)
        substitution = {b: self._blocks.get(b) for b in self._block_names}
        return template.render(**substitution)

    def save_file(self, filename=None):
        if not (filename):
            filename = self.name + ".py"
        code = self.generate_code()
        with open(filename, "w") as f:
            f.write(code)

def renko_plot(data, bricks_col="brick"):
    """
    Draw a renko plot from the given dataframe
    data
        dataframe containing renko data
    bricks_col
        name of the bricks column; default brick
    Note
    -----
    Brick size is calculated from bricks column automatically
    """
    from bokeh.plotting import figure

    data = data.copy()
    brick_size = abs(data[bricks_col].iloc[0] - data[bricks_col].iloc[1])
    data["left"] = range(len(data))
    data["right"] = data.left.values + 1
    data["top"] = data[bricks_col].values
    data["bottom"] = data.top + brick_size
    data["move"] = (data.brick > data.brick.shift(1)) + 0
    data["color"] = ["green" if x == 1 else "red" for x in data.move.values]
    p = figure(title="Renko chart")
    p.quad(
        top="top",
        bottom="bottom",
        left="left",
        right="right",
        color="color",
        source=data,
    )
    return p

class DayTrading:
    def __init__(self, data=None, interval=None, tradebook=None, tradebook_args={}):
        """
        arguments to be passed to init
        data
            primary dataframe
            this is the dataframe on which all work would be done
        interval
            interval as pandas dataframe
        tradebook
            a valid tradebook function
        tradebook_args
            extra argument to be passed to tradebook
        """
        self._interval = interval
        self._data = data
        self._sources = {}
        self._tradebook = tradebook
        self._tradebook_args = tradebook_args
        import pyfolio as pf

        self.pf = pf

    @property
    def data(self):
        return self._data

    @staticmethod
    def agged(data, interval="5min", column_name="timestamp"):
        return (
            data.set_index(column_name)
            .resample(interval, label="right")
            .agg({"open": "first", "high": "max", "low": "min", "close": "last"})
        )

    def _by_day(self):
        """
        Run the function on each day
        """
        func_spec = inspect.getfullargspec(self._tradebook)
        columns = self.data.columns

        def f(data):
            kwargs = {}
            for arg in func_spec.args:
                if arg in columns:
                    if arg in func_spec.annotations.keys():
                        kwargs[arg] = data[arg].values[0]
                    else:
                        kwargs[arg] = data[arg].values
            kwargs.update(self._tradebook_args)
            return self._tradebook(**kwargs)

        grouped = self.data.groupby("date")
        tbs = grouped.apply(f)
        trades = []
        for v in tbs.values:
            trades.extend(v.all_trades)
        return trades
        # This is excess to be corrected
        all_trades = pd.DataFrame(trades)
        all_trades["date"] = pd.to_datetime(all_trades.ts.dt.date)
        all_trades["value"] = all_trades.eval("price*qty*-1")
        return all_trades.tail()

    def _convert_to_legs(self, result=None):
        """
        Convert trades to daily legs for better summary
        result
            a tradebook with alternate buy and sell trades
        """
        if result is None:
            result = self._result
        x = range(0, len(result), 2)
        y = range(1, len(result), 2)
        trds = []
        for m, n in zip(x, y):
            a = result[m]
            b = result[n]
            d = {}
            d["symbol"] = a["symbol"]
            d["order"] = a["order"]
            d["entry_time"] = a["ts"]
            d["entry_price"] = a["price"]
            d["qty"] = a["qty"]
            d["exit_time"] = b["ts"]
            d["exit_price"] = b["price"]
            trds.append(d)
        trds = pd.DataFrame(trds)
        trds["entry_time"] = pd.to_datetime(trds.entry_time)
        trds["exit_time"] = pd.to_datetime(trds.exit_time)
        trds["hour"] = trds.entry_time.dt.hour
        trds["date"] = pd.to_datetime(trds.entry_time.dt.date)
        trds["pnl"] = trds.eval("(exit_price-entry_price)*qty")
        return trds

    @property
    def summary(self):
        return self._summary

    def add_source(self, name: str, data: str):
        """
        adds a data source to the existing sources.
        you could access this as an attribute
        name
            the name of the data source
        data
            actual data as a pandas dataframe
        Note
        -----
        This is just a convenient function to
        add a datasource as an attribute
        """
        self._sources[name] = data
        setattr(self, name, data)

    def perf_stats(self, cost=0, capital=1000):
        returns = (self.summary.set_index("date").pnl - cost) / capital
        return self.pf.timeseries.perf_stats(returns)

    def drawdown_table(self, cost=0, capital=1000):
        returns = (self.summary.set_index("date").pnl - cost) / capital
        return self.pf.timeseries.gen_drawdown_table(returns)

    def run(self):
        print("Started running the program")
        self._result = self._by_day()
        summary = self._convert_to_legs()
        self._summary = summary

def single_filter(frame, col1, col2, func=np.mean):
    """
    Create a single filter and returns results as a dictionary
    frame
        dataframe
    col1
        column on which the filter is to be applied
    col2
        list of columns with each column being a categorical
    func
        arbitary function to be applied on each group
    """
    collect = {}
    for col in col2:
        val = frame.groupby(col)[col1].apply(func)
        collect[col] = val
    return collect

@njit
def clean_ticks(price, threshold=10):
    """
    Clean out of sample ticks
    """
    length = len(price)
    arr = np.zeros(length)
    s = price[0]
    dropped = 0
    nobs = 0
    for i in np.arange(1, length):
        if np.abs(price[i] - s) < threshold:
            arr[i] = price[i]
            s = price[i]
            nobs = 0
        else:
            nobs += 1
            dropped += 1
            if nobs > 10:
                s = price[i]
    return arr[arr > 0]

class WalkForward:
    def __init__(self, data, lb=120, rb=30, factor=None, column=None):
        from collections import defaultdict

        self.data = data
        self.lb = lb
        self.rb = rb
        self._splits = defaultdict(list)
        self._factor = factor
        self._column = column
        self._results = []
        self.conf = []
        self.forward = []

    @property
    def factor(self):
        return self._factor

    @property
    def column(self):
        return self._column

    def get_splits(self):
        return self._splits

    def _generate_splits(self):
        lb, rb = self.lb, self.rb
        indexes = range(lb, len(self.data), rb)
        for index in indexes:
            train = self.data.iloc[index - lb : index]
            test = self.data.iloc[index : index + rb]
            self._splits["train"].append(train)
            self._splits["test"].append(test)

    def set_factor(self, factor):
        """
        The factor should be a categorical column in data
        You can also provide a list of category columns
        """
        if factor in self.data.columns:
            self._factor = factor
            return self._factor
        else:
            return "Factor not found"

    def set_column(self, column):
        """
        The column to test
        """
        if column in self.data.columns:
            self._column = column
            return self._column
        else:
            return "Column not found"

    def run(self):
        """
        Run all tests
        """
        fac = self.factor
        col = self.column
        self._generate_splits()
        splits = self.get_splits()
        for train in splits["train"]:
            t1 = train.groupby(fac)[col].agg(["size", "mean"]).to_dict("index")
            self._results.append(t1)
        for test in splits["test"]:
            t2 = test.groupby(fac)[col].agg(["size", "mean"]).to_dict("index")
            self.forward.append(t2)

    def run_conf(self):
        """
        Run confidence test
        """
        print("Running conf")
        train = self.get_splits()["train"]
        results = self._results
        for data, res in zip(train, results):
            tup = []
            for k, v in res.items():
                s = run_simulation(data, size=v["size"], column=self.column)
                conf = len(s[s > v["mean"]]) / 1000
                tup.append((k, v["mean"], conf))
            self.conf.append(tup)


"""Base document class."""
import contextlib
import warnings
from collections import OrderedDict

from bson import ObjectId, SON

from aiomongodel.errors import ValidationError
from aiomongodel.queryset import MotorQuerySet
from aiomongodel.fields import Field, ObjectIdField, SynonymField, _Empty
from aiomongodel.utils import snake_case

class Meta:
    """Storage for Document meta info.

    Attributes:
        collection_name: Name of the document's db collection (note that
            it should be specified as ``collection`` Meta class attribute).
        indexes: List of ``pymongo.IndexModel`` for collection.
        queryset: Query set class to query documents.
        default_query: Each query in query set will be extended using this
            query through ``$and`` operator.
        default_sort: Default sort expression to order documents in ``find``.
        fields: OrderedDict of document fields as ``{field_name => field}``.
        fields_synonyms: Dict of synonyms for field
            as ``{field_name => synonym_name}``.
        codec_options: Collection's codec options.
        read_preference: Collection's read preference.
        write_concern: Collection's write concern.
        read_concern: Collection's read concern.

    """

    OPTIONS = {'queryset', 'collection',
               'default_query', 'default_sort',
               'fields', 'fields_synonyms', 'indexes',
               'codec_options', 'read_preference', 'read_concern',
               'write_concern'}

    def __init__(self, **kwargs):
        self._validate_options(kwargs)

        self.queryset = kwargs.get('queryset', None)
        self.collection_name = kwargs.get('collection', None)
        self.default_query = kwargs.get('default_query', {})
        self.default_sort = kwargs.get('default_sort', None)
        self.fields = kwargs.get('fields', None)
        self.fields_synonyms = kwargs.get('fields_synonyms', None)
        self.indexes = kwargs.get('indexes', None)

        self.codec_options = kwargs.get('codec_options', None)
        self.read_preference = kwargs.get('read_preference', None)
        self.read_concern = kwargs.get('read_concern', None)
        self.write_concern = kwargs.get('write_concern', None)

    def _validate_options(self, kwargs):
        keys = set(kwargs.keys())
        diff = keys - self.__class__.OPTIONS
        if diff:
            # TODO: change Exception type
            raise ValueError(
                'Unrecognized Meta options: {0}.'.format(', '.join(diff)))

    def collection(self, db):
        """Get collection for documents.

        Args:
            db: Database object.

        Returns:
            Collection object.
        """
        return db.get_collection(
            self.collection_name,
            read_preference=self.read_preference,
            read_concern=self.read_concern,
            write_concern=self.write_concern,
            codec_options=self.codec_options)

class BaseDocumentMeta(type):
    """Base metaclass for documents."""

    meta_options_class = Meta

    def __new__(mcls, name, bases, namespace):
        """Create new Document class.

        Gather meta options, gather fields, set document's meta.
        """
        new_class = super().__new__(mcls, name, bases, namespace)

        if name in {'Document', 'EmbeddedDocument'}:
            return new_class

        # prepare meta options from Meta class of the new_class if any.
        options = mcls._get_meta_options(new_class)

        # gather fields
        (options['fields'],
         options['fields_synonyms']) = mcls._get_fields(new_class)

        setattr(new_class, 'meta', mcls.meta_options_class(**options))

        return new_class

    @classmethod
    def _get_fields(mcls, new_class):
        """Gather fields and fields' synonyms."""
        # we should search for fields in all bases classes in reverse order
        # than python search for attributes so that fields could be
        # overwritten in subclasses.
        # As bases we use __mro__.
        fields = OrderedDict()
        synonyms = dict()
        # there are no fields in base classes.
        ignore_bases = {'object', 'BaseDocument',
                        'Document', 'EmbeddedDocument'}
        mro_ns_gen = (cls.__dict__
                      for cls in reversed(new_class.__mro__)
                      if cls.__name__ not in ignore_bases)

        for ns in mro_ns_gen:
            for name, item in ns.items():
                if isinstance(item, Field):
                    if not item.name:
                        item.name = name
                    fields[name] = item
                elif isinstance(item, SynonymField):
                    synonyms[item] = name

        return fields, {item.original_field_name: name
                        for item, name in synonyms.items()}

    @classmethod
    def _get_meta_options(mcls, new_class):
        """Get meta options from Meta class attribute."""
        doc_meta_options = {}
        doc_meta = new_class.__dict__.get('Meta', None)
        if doc_meta:
            doc_meta_options = {key: doc_meta.__dict__[key]
                                for key in doc_meta.__dict__
                                if not key.startswith('__')}

        return doc_meta_options

class DocumentMeta(BaseDocumentMeta):
    """Document metaclass.

    This meta class add ``_id`` field if it is not specified in
    document class.

    Set collection name for document to snake case of the document class name
    if it is not specified in Meta class attribute of a the document class.

    Attributes:
        queryset: Default query set class.
        default_id_field: Field to use as ``_id`` field if it is not
            specified in document class.

    """

    queryset = MotorQuerySet
    default_id_field = ObjectIdField(name='_id', required=True,
                                     default=lambda: ObjectId())

    @classmethod
    def _get_fields(mcls, new_class):
        fields, synonyms = super()._get_fields(new_class)

        # add _id field if needed
        if '_id' not in fields:
            fields['_id'] = mcls.default_id_field
            setattr(new_class, '_id', mcls.default_id_field)
            return fields, synonyms

        if not fields['_id'].required:
            raise ValueError(
                "'{0}._id' field should be required.".format(
                    new_class.__name__))

        return fields, synonyms

    @classmethod
    def _get_meta_options(mcls, new_class):
        meta_options = super()._get_meta_options(new_class)

        if 'collection' not in meta_options:
            meta_options['collection'] = snake_case(new_class.__name__)

        if 'queryset' not in meta_options:
            meta_options['queryset'] = mcls.queryset

        return meta_options

class EmbeddedDocumentMeta(BaseDocumentMeta):
    """Embedded Document metaclass."""

class BaseDocument(object):
    """Base class for Document and EmbeddedDocument."""

    def __init__(self, *, _empty=False, **kwargs):
        """Initialize document.

        Args:
            _empty (bool): If True return an empty document without setting
                any field.
            **kwargs: Fields values to set. Each key should be a field name
                not a mongo name of the field.
        """
        self._data = OrderedDict()
        if _empty:
            return

        for field_name, field in self.meta.fields.items():
            try:
                value = self._get_field_value_from_data(kwargs, field_name)
            except KeyError:
                value = field.default

            if value is not _Empty:
                setattr(self, field_name, value)

    def to_data(self):
        """Return internal data of the document.

        .. note::
            Internal data can contain embedded document objects,
            lists etc.

        Returns:
            OrderedDict: Data of the document.
        """
        # TODO: Add recursive parameter
        return self._data

    def _get_field_value_from_data(self, data, field_name):
        """Retrieve value from data for given field_name.

        Try use synonym name if field's name is not in data.

        Args:
            data (dict): Data in form {field_name => value}.
            field_name (str): Field's name.

        Raises:
            KeyError: If there is no value in data for given field_name.
        """
        with contextlib.suppress(KeyError):
            return data[field_name]
        # try synonym name
        return data[self.meta.fields_synonyms[field_name]]

    def _set_mongo_data(self, data):
        """Set document's data using mongo data."""
        self._data = OrderedDict()
        for field_name, field in self.meta.fields.items():
            with contextlib.suppress(KeyError):  # ignore missed fields
                self._data[field_name] = field.from_mongo(
                    data[field.mongo_name])

        return self

    def populate_with_data(self, data):
        """Populate document object with data.

        Args:
            data (dict): Document data in form {field_name => value}.

        Returns:
            Self instance.

        Raises:
            AttributeError: On wrong field name.
        """
        # TODO: should we ignore wrong field_names?
        for field_name, value in data.items():
            setattr(self, field_name, value)
        return self

    def to_mongo(self):
        """Convert document to mongo format."""
        son = SON()
        for field_name, field in self.meta.fields.items():
            if field_name in self._data:
                son[field.mongo_name] = field.to_mongo(self._data[field_name])

        return son

    @classmethod
    def from_mongo(cls, data):
        """Create document from mongo data.

        Args:
            data (dict): SON data loaded from mongodb.

        Returns:
            Document instance.
        """
        inst = cls(_empty=True)
        inst._set_mongo_data(data)
        return inst

    @classmethod
    def from_data(cls, data):
        """Create document from user provided data.

        Args:
            data (dict): Data dict in form {field_name => value}.

        Returns:
            Document isinstance.
        """
        return cls(**data)

    def validate(self):
        """Validate data.

        Returns:
            Self instance.

        Raises:
            ValidationError: If document's data is not valid.
        """
        self.__class__.validate_document(self)
        return self

    @classmethod
    def validate_document(cls, document):
        """Validate given document.

        Args:
            document: Document instance to validate.

        Raises:
            ValidationError: If document's data is not valid.
        """
        errors = {}
        for field_name, field in cls.meta.fields.items():
            try:
                field.validate(document._data[field_name])
            except ValidationError as e:
                errors[field_name] = e
            except KeyError:
                if field.required:
                    errors[field_name] = ValidationError('field is required')

        if errors:
            raise ValidationError(errors)

class Document(BaseDocument, metaclass=DocumentMeta):
    """Base class for documents.

    Each document class should be defined by inheriting from this
    class and specifying fields and optionally meta options using internal
    Meta class.

    Fields are inherited from base classes and can be overwritten.

    Meta options are NOT inherited.

    Possible meta options for ``class Meta``:

    - ``collection``: Name of the document's db collection.
    - ``indexes``: List of ``pymongo.IndexModel`` for collection.
    - ``queryset``: Query set class to query documents.
    - ``default_query``: Each query in query set will be extended using
      this query through ``$and`` operator.
    - ``default_sort``: Default sort expression to order documents in
      ``find``.
    - ``codec_options``: Collection's codec options.
    - ``read_preference``: Collection's read preference.
    - ``write_concern``: Collection's write concern.
    - ``read_concern``: Collection's read concern.

    .. note::
        Indexes are not created automatically. Use
        ``MotorQuerySet.create_indexes`` method to create document's indexes.

    Example:

    .. code-block:: python

        from pymongo import IndexModel, ASCENDING, DESCENDING

        class User(Document):
            name = StrField(regex=r'^[a-zA-Z]{6,20}$')
            is_active = BoolField(default=True)
            created = DateTimeField(default=lambda: datetime.utcnow())

            class Meta:
                # define a collection name
                collection = 'users'
                # define collection indexes. Use
                # await User.q(db).create_indexes()
                # to create them on application startup.
                indexes = [
                    IndexModel([('name', ASCENDING)], unique=True),
                    IndexModel([('created', DESCENDING)])]
                # order by `created` field by default
                default_sort = [('created', DESCENDING)]

        class ActiveUser(User):
            is_active = BoolField(default=True, choices=[True])

            class Meta:
                collection = 'users'
                # specify a default query to work ONLY with
                # active users. So for example
                # await ActiveUser.q(db).count({})
                # will count ONLY active users.
                default_query = {'is_active': True}

    """

    @classmethod
    def q(cls, db, session=None):
        """Return queryset object.

        Args:
            db: Motor database object.
            session: Motor client session object.

        Returns:
            MotorQuerySet: Queryset object.
        """
        return cls.meta.queryset(cls, db, session=session)

    @classmethod
    def coll(cls, db):
        """Return raw collection object.

        Args:
            db: Motor database object.

        Returns:
            MotorCollection: Raw Motor collection object.
        """
        return cls.meta.collection(db)

    @classmethod
    async def create(cls, db, session=None, **kwargs):
        """Create document in mongodb.

        Args:
            db: Database instance.
            session: Motor session object.
            **kwargs: Document's fields values.

        Returns:
            Created document instance.

        Raises:
            ValidationError: If some fields are not valid.
        """
        warnings.warn("Use `create` method of `queryset` instead",
                      DeprecationWarning,
                      stacklevel=2)

        inst = cls.from_data(kwargs)
        return await inst.save(db, do_insert=True, session=session)

    async def save(self, db, do_insert=False, session=None):
        """Save document in mongodb.

        Args:
            db: Database instance.
            do_insert (bool): If ``True`` always perform ``insert_one``, else
                perform ``replace_one`` with ``upsert=True``.
            session: Motor session object.
        """
        data = self.to_mongo()
        if do_insert:
            await self.__class__.q(db, session=session).insert_one(data)
        else:
            await self.__class__.q(db, session=session).replace_one(
                    {'_id': data['_id']},
                    data, upsert=True)

        return self

    async def reload(self, db, session=None):
        """Reload current object from mongodb."""
        cls = self.__class__
        data = await cls.coll(db).find_one(self.query_id, session=session)
        self._set_mongo_data(data)
        return self

    async def update(self, db, update_document, session=None):
        """Update current object using query.

        Usage:

        .. code-block:: python

            class User(Document):
                name = StrField()
                value = IntField(default=0)

            async def go(db):
                u = await User(name='xxx').save(db)
                await u.update(db,
                               {'$set': {User.name.s: 'yyy'},
                                '$inc': {User.value.s: 1}})

        """
        qs = self.__class__.q(db, session=session)
        count = await qs.update_one(self.query_id, update_document)
        # TODO: maybe we should return number of updates or raise if it's 0.
        if count > 0:
            await self.reload(db, session=session)

        return self

    async def delete(self, db, session=None):
        """Delete current object from db."""
        qs = self.__class__.q(db, session=session)
        return await qs.delete_one(self.query_id)

    @property
    def query_id(self):
        return {'_id': self.__class__._id.to_mongo(self._id)}

    @classmethod
    async def create_collection(self, db, session=None):
        """Create collection for documents.

        Args:
            db: Database object.

        Returns:
            Collection object.
        """
        return await db.create_collection(
            self.meta.collection_name,
            read_preference=self.meta.read_preference,
            read_concern=self.meta.read_concern,
            write_concern=self.meta.write_concern,
            codec_options=self.meta.codec_options,
            session=session)

class EmbeddedDocument(BaseDocument, metaclass=EmbeddedDocumentMeta):
    """Base class for embedded documents."""


# -*- coding: utf-8 -*-

import requests
import json
import xmltodict

"""
amount（） src - currency（）dest - currency（）reference - date（）the output of 2 parameters
amount（） currency（）
"""
def currency_exchange(amount=100, currency='USD', dest_currency='JPY', reference_date='2020-03-03'):
    content = requests.get('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml', timeout=30).content
    ceDict = {}
    for d in xmltodict.parse(content)["gesmes:Envelope"]["Cube"]["Cube"]:
        ceDict[d['@time']] = dict([(dd["@currency"], dd["@rate"]) for dd in d['Cube']])

    a = ceDict.get(reference_date, {}).get(currency.upper())
    b = ceDict.get(reference_date, {}).get(dest_currency.upper())
    if a and b:
        amount2 = round(amount * float(a) / float(b), 4)
        print(u"dest-amount: %s, dest-currency: %s, rate: %s -> %s" % (amount2, dest_currency, a, b))
        return amount2, dest_currency
    else:
        print(u"wrong parameter\n%s" % json.dumps(ceDict, indent=4, ensure_ascii=False))

currency_exchange(dest_currency='bgn')


from __future__ import print_function

import pp
import logging
import sys
import argparse
import glob
import os

def run_task( benchmark, command, domain, instance, folder, timeout ) :
    import benchmark
    import os
    import json

    current_dir = os.getcwd()
    os.chdir( folder )
    log = 'trace.log'
    rv, time = benchmark.run( command, timeout, 4096, benchmark.Log(log) )
    command_info = {}
    command_info['exit_code'] = rv
    command_info['wall_time'] = time

    if rv == 0 :
        json_search_data = 'search.json'
        if os.path.exists( json_search_data ) :
            with open( json_search_data ) as input :
                try :
                    s = input.read()
                    data = json.loads( s )
                except ValueError as e:
                    print( e )
                    print( "Read from file:")
                    print( s )
                    os.chdir( current_dir )
                    return
                for k, v in data.items() :
                    command_info[k] = v
        else :
            print("No json data payload found!")

    os.chdir( current_dir )
    data_folder = 'data'
    data_folder = os.path.join( data_folder, domain )
    if not os.path.exists( data_folder ) :
        os.makedirs(data_folder)

    info_filename  = os.path.join(data_folder,'{0}.json'.format(instance))
    if not os.path.exists( info_filename ) :
        info = {}
        info['domain'] = domain
        info['instance'] = instance
        info['planners_data'] = {}
    else :
        with open( info_filename ) as instream :
            info = json.loads( instream.read() )
    info['planners_data'][command] = command_info

    with open ( info_filename, 'w' ) as outstream:
        payload = json.dumps( info, separators=(',',':'), indent=4, sort_keys = True)
        outstream.write( payload )

    return payload

def main() :

    parser = argparse.ArgumentParser( description = "Driver script for running planner(s) over benchmarks")
    parser.add_argument( '--benchmark', required=True, help='Path to benchmarks')
    parser.add_argument( '--domain', required=False, help='Name of domain to be tested')
    parser.add_argument( '--timeout', required=False, help='Planner time out (defaults to 1800 secs)')
    parser.add_argument( '--cpus', required=False, help='Number of cpus to be used (defaults to 2)')
    parser.add_argument( '--secret', required=True, help='Server password')
    parser.add_argument( '--planner', required=True, help='Command used to invoke the planner')

    args = parser.parse_args()

    if not os.path.exists( args.benchmark ) :
        print ( 'Could not find directory {0}'.format(args.benchmark), file=sys.stderr )
        sys.exit(1)

    if args.timeout is None :
        args.timeout = 1800

    print(args.planner)

    tasks = []
    if args.domain is not None :
        domain_dirs = [ os.path.join( args.benchmark, args.domain ) ]
    else :
        domain_dirs = glob.glob( os.path.join( args.benchmark, '*'))
        domain_dirs = [ dom for dom in domain_dirs if os.path.isdir(dom) ]

    for domain_dir in domain_dirs :
        if not os.path.exists( domain_dir ) :
            print( 'Could not find domain directory {0}'.format( domain_dir ), file=sys.stderr)
            sys.exit(1)
        instances = glob.glob( os.path.join( domain_dir, '*') )
        instances = [ inst for inst in instances if os.path.isdir( inst ) ]
        for inst in instances :
            tasks.append( (os.path.basename(args.benchmark), args.planner, os.path.basename(domain_dir), os.path.basename(inst), inst, args.timeout) )
    if args.cpus is None :
        num_cpus = 2
    else :
        num_cpus = int(args.cpus)
    ppservers = ()
    job_server = pp.Server( num_cpus, ppservers=ppservers, socket_timeout=2, secret=args.secret)
    print ('Tasks to be executed: {0}'.format(len(tasks)))
    jobs = [  (task, job_server.submit( run_task, (task), (), ())) for task in tasks ]

    for input, job in jobs :
        print( 'Output of {0} is {1}'.format( input, job() ) )

    job_server.print_stats()

if __name__ == '__main__' :
    main()


import logging
import unittest
from typing import Hashable, List, Tuple

import numpy as np

from reach import Reach

logger = logging.getLogger(__name__)

class TestVectorize(unittest.TestCase):
    def data(self) -> Tuple[List[Hashable], np.ndarray]:
        words: List[Hashable] = [
            "donatello",
            "leonardo",
            "raphael",
            "michelangelo",
            "splinter",
            "hideout",
        ]
        random_generator = np.random.RandomState(seed=44)
        vectors = random_generator.standard_normal((6, 50))

        return words, vectors

    def test_vectorize_no_unk(self) -> None:
        words, vectors = self.data()
        reach = Reach(vectors, words)

        with self.assertRaises(ValueError):
            reach.vectorize(("donatello", "abcd"), remove_oov=False)

        with self.assertRaises(ValueError):
            reach.vectorize([])

        with self.assertRaises(ValueError):
            reach.vectorize("")

        vec = reach.vectorize(("donatello", "abcd"), remove_oov=True)
        self.assertEqual(len(vec), 1)
        self.assertTrue(np.allclose(vec, reach["donatello"]))

    def test_vectorize_unk(self) -> None:
        words, vectors = self.data()
        words.append("<UNK>")
        vectors = np.concatenate([vectors, np.zeros((1, vectors.shape[1]))])
        reach = Reach(vectors, words, unk_index=len(words) - 1)

        self.assertEqual(reach.indices[reach.unk_index], "<UNK>")  # type: ignore
        self.assertTrue(np.allclose(vectors[-1], np.zeros(reach.size)))

    def test_bow_no_unk(self) -> None:
        words, vectors = self.data()
        reach = Reach(vectors, words)

        bow = reach.bow(["donatello", "leonardo", "michelangelo"])
        self.assertEqual(bow, [0, 1, 3])

        bow = reach.bow(["donatello", "leonardo", "rgieurghegh"], remove_oov=True)
        self.assertEqual(bow, [0, 1])

        with self.assertRaises(ValueError):
            bow = reach.bow(["donatello", "wroughwuorg"], remove_oov=False)

        with self.assertRaises(ValueError):
            bow = reach.bow("")

    def test_bow_unk(self) -> None:
        words, vectors = self.data()
        words.append("<UNK>")
        vectors = np.concatenate([vectors, np.zeros((1, vectors.shape[1]))])
        reach = Reach(vectors, words, unk_index=len(words) - 1)

        bow = reach.bow(["donatello", "leonardo", "rgieurghegh"])
        self.assertEqual(bow, [0, 1, reach.unk_index])

        bow = reach.bow(["donatello", "leonardo", "rgieurghegh"], remove_oov=True)
        self.assertEqual(bow, [0, 1])

    def test_transform(self) -> None:
        words, vectors = self.data()
        reach = Reach(vectors, words)

        with self.assertRaises(ValueError):
            reach.transform([["donatello", "raphael"], ["dog"], ["clown", "donatello"]])

        matrices = reach.transform(
            [["donatello", "raphael"], ["donatello", "splinter"]]
        )

        expected = [
            np.stack([reach["donatello"], reach["raphael"]]),
            np.stack([reach["donatello"], reach["splinter"]]),
        ]
        for matrix, exp_matrix in zip(matrices, expected):
            self.assertTrue(np.allclose(matrix, exp_matrix))

        matrices = reach.transform(
            [["donatello", "raphael"], ["rqghqgr", "splinter"]], remove_oov=True
        )

        expected = [
            np.stack([reach["donatello"], reach["raphael"]]),
            np.stack([reach["splinter"]]),
        ]
        for matrix, exp_matrix in zip(matrices, expected):
            self.assertTrue(np.allclose(matrix, exp_matrix))

        with self.assertRaises(ValueError):
            reach.transform([[]])

        empty_result = reach.transform([])
        self.assertEqual(empty_result, [])

    def test_mean_pool(self) -> None:
        words, vectors = self.data()
        reach = Reach(vectors, words)

        with self.assertRaises(ValueError):
            reach.mean_pool(["donatello", "dog"])

        vec = reach.mean_pool(["donatello", "dog"], safeguard=False)
        self.assertTrue(np.allclose(vec, np.zeros_like(vec)))

        vec = reach.mean_pool(["donatello", "dog"], remove_oov=True)
        self.assertTrue(np.allclose(vec, reach["donatello"]))

    def test_mean_pool_unk(self) -> None:
        words, vectors = self.data()
        words.append("<UNK>")
        vectors = np.concatenate([vectors, np.zeros((1, vectors.shape[1]))])
        reach = Reach(vectors, words, unk_index=len(words) - 1)

        vec = reach.mean_pool(["donatello", "dog"])
        self.assertTrue(np.allclose(vec, reach["donatello"] / 2))

        vec = reach.mean_pool(["donatello", "dog"], safeguard=True)
        self.assertTrue(np.allclose(vec, reach["donatello"] / 2))

        vec = reach.mean_pool(["donatello", "dog"], remove_oov=True)
        self.assertTrue(np.allclose(vec, reach["donatello"]))

        vec = reach.mean_pool([], safeguard=False)
        self.assertTrue(np.allclose(vec, np.zeros_like(vec)))

        with self.assertRaises(ValueError):
            reach.mean_pool_corpus([[], ["dog"], ["guogrwohu"]], safeguard=True)

        matrix = reach.mean_pool_corpus([[], ["dog"], ["guogrwohu"]], safeguard=False)
        self.assertTrue(np.allclose(matrix, np.zeros_like(matrix)))


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

def classify(X,y):
    names = [
        "Nearest Neighbors",
        "Linear SVM",
        "RBF SVM",
        "Gaussian Process",
        "Decision Tree",
        "Random Forest",
        "Neural Net",
        "AdaBoost",
        "Naive Bayes",
        "QDA",
    ]

    classifiers = [
        KNeighborsClassifier(3),
        SVC(kernel="linear", C=0.025),
        SVC(gamma=2, C=1),
        GaussianProcessClassifier(1.0 * RBF(1.0)),
        DecisionTreeClassifier(max_depth=5),
        RandomForestClassifier(max_depth=5, n_estimators=10, max_features=1),
        MLPClassifier(alpha=1, max_iter=1000),
        AdaBoostClassifier(),
        GaussianNB(),
        QuadraticDiscriminantAnalysis(),
    ]

    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.4, random_state=42
    )

    # iterate over classifiers
    for name, clf in zip(names, classifiers):
        clf.fit(X_train, y_train)
        score = clf.score(X_test, y_test)

        print(name, score, flush=True)

if __name__ == "__main__":
    with open('merged.txt') as f:
        df = pd.read_csv(f,sep='\t',skiprows=1)

    df = df.drop(['NCBI_tax_id', 'clade_name'], axis=1)
    X = df.transpose().to_numpy().tolist()
    y = [0]*28 + [1]*31

    classify(X,y)


"""More detailed and complicated unit properties"""

from model.unit import Unit

from typing import Optional
from debug_interface import DebugInterface
from debugging.color import Color

from strategy.float import equal_float, lt_float, le_float
from strategy.geometry import Vec, Line, Segment, Circle

class UnitAdv:
    """Unit and its properties"""

    unit: Unit
    weapons: dict
    ticks_per_seconds: float
    max_unit_forward_speed: float
    max_unit_backward_speed: float
    unit_acceleration: float
    speed_limit_circle: Circle
    acceleration_limit_circle: Circle

    def __init__(self, unit: Unit, weapons: dict, max_unit_forward_speed: float, max_unit_backward_speed: float,
                 unit_acceleration: float, ticks_per_seconds: float):
        self.ticks_per_seconds = ticks_per_seconds
        """Ticks per seconds"""
        self.unit = unit
        """Unit itself"""

        self.velocity = (Vec() + self.unit.velocity) / self.ticks_per_seconds
        max_unit_forward_speed /= self.ticks_per_seconds
        max_unit_backward_speed /= self.ticks_per_seconds
        unit_acceleration /= self.ticks_per_seconds

        self.weapons = weapons
        """Available weapons"""

        self.max_unit_forward_speed = max_unit_forward_speed * (
                1 - (1 - weapons[
                    self.unit.weapon].aim_movement_speed_modifier) * self.unit.aim)
        """Maximal unit forward speed at this moment"""

        self.max_unit_backward_speed = max_unit_backward_speed * (
                1 - (1 - weapons[
                    self.unit.weapon].aim_movement_speed_modifier) * self.unit.aim)
        """Maximal unit backward speed at this moment"""

        self.unit_acceleration = unit_acceleration
        """Unit acceleration"""

        self.speed_limit_circle = Circle(
            (Vec() + self.unit.direction) * (
                    (self.max_unit_forward_speed - self.max_unit_backward_speed) / 2) + self.unit.position,
            (self.max_unit_forward_speed + self.max_unit_backward_speed) / 2)
        """Speed limit circle, it limits target speed"""

        self.acceleration_limit_circle = Circle(
            Vec() + self.velocity + self.unit.position,
            self.unit_acceleration / 30)
        """Limit of next speed"""

    def move_to_position(self, position: Vec, debug: Optional[DebugInterface]) -> Vec:
        """Returns velocity vector to move in several steps to 'position'"""

        vec_unit_position = position - self.unit.position
        """Vector from unit.position to 'position'"""

        # if debug is not None:
        #     debug.add_segment(self.unit.position, position, 0.5, Color(0.5, 0.5, 1, 1))

        if position in self.acceleration_limit_circle:
            """'position' in acceleration_limit_circle that means TargetVector equals vec_unit_to_position"""

            if debug is not None:
                print("First_move_to")

            return vec_unit_position

        if position in self.speed_limit_circle:
            """'position' in speed_limit_circle that means TargetVector equals
                vector from unit.position to point(acceleration_limit_circle ⋂ 'position')"""

            intersection_acceleration_position = \
                self.acceleration_limit_circle.intersection_with_segment(position=position)[0]
            """Intersection acceleration_limit_circle with
                vector form acceleration_limit_circle.centre_pos to 'position'"""

            if debug is not None:

                print("Second_move_to")

            return intersection_acceleration_position - self.unit.position

        """'position' is out of any of circles that means TargetVector equals some vector.. beginning at unit.position
            and ending at some point on acceleration_limit_circle..."""

        intersection_speed_circle_vec_unit_position = \
            self.speed_limit_circle.intersection_with_segment(position=position)[0]
        """Intersection of speed_limit_circle and vec_unit_position"""

        intersection_acceleration_previous_intersection = \
            self.acceleration_limit_circle.intersection_with_segment(
                position=intersection_speed_circle_vec_unit_position)[0]
        """Intersection of acceleration_limit_circle and intersection_speed_circle_vec_unit_position"""

        if debug is not None:
            print("Third_move_to")

        if intersection_acceleration_previous_intersection is not None:
            return intersection_acceleration_previous_intersection - self.unit.position
        print("NONENONENONENONENONENONENONENONENONENONENONENONENONENONENONENONENONE")
        return Vec() + self.unit.position

    def turn_to_position(self, position: Vec) -> Vec:
        """Returns direction(vector of length 1) to turn in several steps
            centre of view_field in the direction of the position"""

        new_direction = position - self.unit.position
        """New vector direction(any length)"""

        if new_direction != Vec() and lt_float(new_direction.len, 0.6):
            """Checking and recounting vector length that it follows the rules"""
            new_direction *= (1 / new_direction.len)

        return new_direction


import pexpect
import re
import sys
import telnetlib
import logging

class TelConFailed(Exception):
    pass

class TelCon:
    def __init__(self, logger = logging.getLogger(__name__)):
        self.logger = logger
        logging.basicConfig(level=logging.DEBUG)
        self.tel = pexpect.spawn("telnet localhost 4444", timeout = 3)
        telcon_start = self.tel.expect([
            "Connected to localhost",
            "unable to"
            ])
        if telcon_start == 0:
            logging.info("connected to telnet")
        else:
            raise TelConFailed()

    def __enter__(self):
        return self

    def halt(self):
        self.tel.sendline("reset halt")
        self.tel.expect("[\s\S]*target halted[\s\S]*")
        logging.info("halted")

    def remove_protection(self):
        self.tel.sendline("flash protect 0 0 last off")
        self.tel.expect("cleared protection for sectors")
        logging.info("erased")

    def erase(self):
        self.tel.sendline("flash erase_sector 0 0 0")
        self.tel.expect("[\s\S]*erased sectors[\s\S]*")
        logging.info("erased")

    def load(self):
        self.tel.sendline("flash write_image erase /root/workspace/Provisioning-jig-Debugger/debug/DFU_ST-Link-V2.hex")
        self.tel.expect("[\s\S]*wrote[\s\S]*from file[\s\S]*")
        self.tel.sendline("verify_image /root/workspace/Provisioning-jig-Debugger/debug/DFU_ST-Link-V2.hex")
        self.tel.expect("[\s\S]*verified[\s\S]*bytes in[\s\S]*")
        logging.info("LOADED!")

    def __exit__(self, type, value, traceback):
        self.tel.send("exit")
        logging.info("Telnet connection terminated")


import os
from K_MC import calculate_centroids_and_clusters, calculate_k, classify
import sys
sys.path.append('../')  # Move path so that we can import from the datasets directory
from datasets.weather import create_data_set, create_labeled_data_set  # noqa: E402

if __name__ == "__main__":
    unlabeled_data_set = create_data_set('../datasets/dataset1.csv')
    labeled_data_set = create_labeled_data_set('../datasets/dataset1.csv')
    validation_set = create_labeled_data_set('../datasets/dataset1.csv')

    # Label un labeled data using the validation set
    centroids, clusters = calculate_centroids_and_clusters(unlabeled_data_set, calculate_k(unlabeled_data_set))

    # Normally k means clustering is used for unlabelled data,
    # but we have an labeled data set.
    # So we can label our found clusters by comparing the given cluster id with the label it has in de data set.
    # NOTE: It could be that our algorithm finds more (or less) clusters than we have labels, so this is not a 100% validation.

    # Get every label for every cluster id
    label_count_by_id = {}
    for data in labeled_data_set:
        label = classify(centroids, data[1])
        if label in label_count_by_id:
            label_count_by_id[label].append(data[0])
        else:
            label_count_by_id[label] = []

    # Count every label and set the cluster label to the most appearing one
    label_by_id = {}
    for key, labels in label_count_by_id.items():
        count = {}
        for label in labels:
            if label in count:
                count[label] += 1
            else:
                count[label] = 0
        count_max = max(count.items(), key = lambda x: x[1])
        label_by_id[key] = count_max[0]

    # Since we have the labels on our data set, we can also validate our data with our validation set
    correct = 0
    for data in validation_set:
        c = label_by_id[classify(centroids, data[1])]
        correct += (c == data[0])
    print(correct/len(validation_set) * 100)  # AVG: 51~%

    # Test the unlabeled data
    unlabeled = [
        (40, 52, 2, 102, 103, 0, 0),
        (25, 48, -18, 105, 72, 6, 1),
        (23, 121, 56, 150, 25, 18, 18),
        (27, 229, 146, 308, 130, 0, 0),
        (41, 65, 27, 123, 95, 0, 0),
        (46, 162, 100, 225, 127, 0, 0),
        (23, -27, -41, -16, 0, 0, -1),
        (28, -78, -106, -39, 67, 0, 0),
        (38, 166, 131, 219, 58, 16, 41),
    ]

    # Label the unlabeled data
    for day in unlabeled:
        print(label_by_id[classify(centroids, day)])

    os.system("pause")


from builtins import str
from builtins import object
import re
import json
from buildtimetrend import logger
from buildtimetrend import tools
from buildtimetrend.buildjob import BuildJob
from buildtimetrend.collection import Collection
from buildtimetrend.travis.connector import TravisOrgConnector
from buildtimetrend.travis.connector import TravisConnector
from buildtimetrend.travis.substage import TravisSubstage
try:
    # For Python 3.0 and later
    from urllib.error import HTTPError, URLError
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import HTTPError, URLError

# strings to parse timestamps in Travis CI log file
TRAVIS_LOG_PARSE_TIMING_STRINGS = [
    r'travis_time:end:(?P<end_hash>.*):start=(?P<start_timestamp>\d+),'
    r'finish=(?P<finish_timestamp>\d+),duration=(?P<duration>\d+)\x0d\x1b',
    r'travis_fold:end:(?P<end_stage>\w+)\.(?P<end_substage>\d+)\x0d\x1b',
    r'travis_fold:start:(?P<start_stage>\w+)\.(?P<start_substage>\d+)\x0d\x1b',
    r'travis_time:start:(?P<start_hash>.*)\x0d\x1b\[0K',
    r'\$\ (?P<command>.*)\r',
]
TRAVIS_LOG_PARSE_WORKER_STRING = r'Using worker:\ (?P<hostname>.*):(?P<os>.*)'

class TravisData(object):

    """Gather data from Travis CI using the API."""

    def __init__(self, repo, build_id, connector=None):
        """
        Retrieve Travis CI build data using the API.

        Parameters:
        - repo : github repository slug (fe. buildtimetrend/python-lib)
        - build_id : Travis CI build id (fe. 158)
        - connector : Travis Connector instance
        """
        self.builds_data = {}
        self.build_jobs = {}
        self.current_build_data = {}
        self.current_job = BuildJob()
        self.travis_substage = None
        self.repo = repo
        self.build_id = str(build_id)
        # set TravisConnector if it is defined
        if isinstance(connector, TravisConnector):
            self.connector = connector
        # use Travis Org connector by default
        else:
            self.connector = TravisOrgConnector()

    def get_build_data(self):
        """
        Retrieve Travis CI build data.

        Returns true if retrieving data was succesful, false on error.
        """
        request = 'repos/{repo}/builds?number={build_id}'.format(
            repo=self.repo, build_id=self.build_id
        )
        try:
            self.builds_data = self.connector.json_request(request)
        except (HTTPError, URLError) as msg:
            logger.error("Error getting build data from Travis CI: %s", msg)
            return False

        # log builds_data
        logger.debug(
            "Build #%s data : %s",
            str(self.build_id),
            json.dumps(self.builds_data, sort_keys=True, indent=2)
        )

        return True

    def get_substage_name(self, command):
        """
        Resolve Travis CI substage name that corresponds to a cli command.

        Parameters:
        - command : cli command
        """
        if not tools.is_string(command):
            return ""

        if len(self.current_build_data) > 0 and \
                "config" in self.current_build_data:
            build_config = self.current_build_data["config"]
        else:
            logger.warning(
                "Travis CI build config is not set"
            )
            return ""

        # check if build_config collection is empty
        if build_config:
            for stage_name, commands in build_config.items():
                if tools.is_list(commands) and command in commands:
                    substage_number = commands.index(command) + 1
                    substage_name = "{stage}.{substage:d}".format(
                        stage=stage_name, substage=substage_number
                    )
                    logger.debug(
                        "Substage %s corresponds to '%s'",
                        substage_name, command
                    )
                    return substage_name

        return ""

    def process_build_jobs(self):
        """
        Retrieve Travis CI build job data.

        Method is a generator, iterate result to get each processed build job.
        """
        if len(self.builds_data) > 0 and "builds" in self.builds_data:
            for build in self.builds_data['builds']:
                self.current_build_data = build

                if "job_ids" in build:
                    for job_id in build['job_ids']:
                        yield self.process_build_job(job_id)

            # reset current_build_data after builds are processed
            self.current_build_data = {}

    def process_build_job(self, job_id):
        """
        Retrieve Travis CI build job data.

        Parameters:
        - job_id : ID of the job to process
        """
        if job_id is None:
            return None

        # retrieve job data from Travis CI
        job_data = self.get_job_data(job_id)
        # process build/job data
        self.process_job_data(job_data)
        # parse Travis CI job log file
        self.parse_job_log(job_id)

        # store build job
        self.build_jobs[str(job_id)] = self.current_job
        # create new build job instance
        self.current_job = BuildJob()

        # return processed build job
        return self.build_jobs[str(job_id)]

    def get_job_data(self, job_id):
        """
        Retrieve Travis CI job data.

        Parameters:
        - job_id : ID of the job to process
        """
        request = 'jobs/{:s}'.format(str(job_id))
        job_data = self.connector.json_request(request)

        # log job_data
        logger.debug(
            "Job #%s data : %s",
            str(job_id),
            json.dumps(job_data, sort_keys=True, indent=2)
        )

        return job_data

    def process_job_data(self, job_data):
        """
        Process Job/build data.

        Set build/job properties :
        - Build/job ID
        - build result : passed, failed, errored
        - git repo
        - git branch
        - CI platform : Travis
        - build matrix (language, language version, compiler, ...)
        - build_trigger : push, pull_request
        - pull_request (is_pull_request, title, number)

        Parameters:
        - job_data : dictionary with Travis CI job data
        """
        self.current_job.add_property(
            "build",
            # buildnumber is part before "." of job number
            job_data['job']['number'].split(".")[0]
        )
        self.current_job.add_property("job", job_data['job']['number'])
        self.current_job.add_property("branch", job_data['commit']['branch'])
        self.current_job.add_property(
            "repo",
            job_data['job']['repository_slug']
        )
        self.current_job.add_property("ci_platform", 'travis')
        self.current_job.add_property("result", job_data['job']['state'])

        self.set_build_matrix(job_data)

        self.process_pull_request_data()

        self.current_job.set_started_at(job_data['job']['started_at'])
        self.current_job.set_finished_at(job_data['job']['finished_at'])

        # calculate job duration from start and finished timestamps
        # if no timing tags are available
        if not self.has_timing_tags():
            self.current_job.add_property("duration", self.get_job_duration())

    def set_build_matrix(self, job_data):
        """
        Retrieve build matrix data from job data and store in properties.

        Properties :
        - language
        - language version (if applicable)
        - compiler (if applicable)
        - operating system
        - environment parameters

        Parameters:
        - job_data : dictionary with Travis CI job data
        """
        # check if job config data exists
        if 'job' not in job_data or 'config' not in job_data['job']:
            logger.warning("Job config data doesn't exist")
            return

        build_matrix = Collection()
        job_config = job_data['job']['config']

        if 'language' in job_config:
            language = job_config['language']
            build_matrix.add_item('language', language)

            # set language version
            # ('d', 'dart', 'go', 'perl', 'php', 'python', 'rust')
            if language in job_config:
                if language == 'android':
                    build_matrix.add_item(
                        "language_components",
                        " ".join(job_config[language]["components"])
                    )
                else:
                    build_matrix.add_item(
                        'language_version',
                        str(job_config[language])
                    )

        # language specific build matrix parameters
        parameters = {
            'ghc': 'ghc',  # Haskell
            'jdk': 'jdk',  # Java, Android, Groovy, Ruby, Scala
            'lein': 'lein',  # Clojure
            'mono': 'mono',  # C#, F#, Visual Basic
            'node_js': 'node_js',  # Javascript
            'otp_release': 'otp_release',  # Erlang
            'rvm': 'rvm',  # Ruby, Objective-C
            'gemfile': 'gemfile',  # Ruby, Objective-C
            'xcode_sdk': 'xcode_sdk',  # Objective-C
            'xcode_scheme': 'xcode_scheme',  # Objective-C
            'compiler': 'compiler',  # C, C++
            'os': 'os',
            'env': 'parameters'
        }
        for parameter, name in parameters.items():
            if parameter in job_config:
                build_matrix.add_item(name, str(job_config[parameter]))

        self.current_job.add_property(
            "build_matrix",
            build_matrix.get_items_with_summary()
        )

    def process_pull_request_data(self):
        """Retrieve pull request data from Travis CI API."""
        # check if collection is empty
        if self.current_build_data:
            if "event_type" in self.current_build_data:
                # build trigger (push or pull_request)
                self.current_job.add_property(
                    "build_trigger",
                    self.current_build_data["event_type"]
                )

            # pull_request
            pull_request_data = {}
            if "pull_request" in self.current_build_data:
                pull_request_data["is_pull_request"] = \
                    self.current_build_data["pull_request"]
            else:
                pull_request_data["is_pull_request"] = False

            if "pull_request_title" in self.current_build_data:
                pull_request_data["title"] = \
                    self.current_build_data["pull_request_title"]

            if "pull_request_number" in self.current_build_data:
                pull_request_data["number"] = \
                    self.current_build_data["pull_request_number"]

            self.current_job.add_property("pull_request", pull_request_data)

    def parse_job_log(self, job_id):
        """
        Parse Travis CI job log.

        Parameters:
        - job_id : ID of the job to process
        """
        self.parse_job_log_stream(self.connector.download_job_log(job_id))

    def parse_job_log_file(self, filename):
        """
        Open a Travis CI log file and parse it.

        Parameters :
        - filename : filename of Travis CI log
        Returns false if file doesn't exist, true if it was read successfully.
        """
        # load timestamps file
        if not tools.check_file(filename):
            return False

        # read timestamps, calculate stage duration
        with open(filename, 'rb') as file_stream:
            self.parse_job_log_stream(file_stream)

        return True

    def parse_job_log_stream(self, stream):
        """
        Parse Travis CI job log stream.

        Parameters:
        - stream : stream of job log file
        """
        self.travis_substage = TravisSubstage()
        check_timing_tags = self.has_timing_tags()

        for line in stream:
            # convert to str if line is bytes type
            if isinstance(line, bytes):
                line = line.decode('utf-8')
            # parse Travis CI timing tags
            if check_timing_tags and 'travis_' in line:
                self.parse_travis_time_tag(line)
            # parse Travis CI worker tag
            if 'Using worker:' in line:
                self.parse_travis_worker_tag(line)

    def parse_travis_time_tag(self, line):
        """
        Parse and process Travis CI timing tags.

        Parameters:
        - line : line from logfile containing Travis CI tags
        """
        if self.travis_substage is None:
            self.travis_substage = TravisSubstage()

        escaped_line = line.replace('\x0d', '*').replace('\x1b', 'ESC')
        logger.debug('line : %s', escaped_line)

        # parse Travis CI timing tags
        for parse_string in TRAVIS_LOG_PARSE_TIMING_STRINGS:
            result = re.search(parse_string, line)
            if result:
                self.travis_substage.process_parsed_tags(result.groupdict())

                # when finished : log stage and create a new instance
                if self.travis_substage.has_finished():
                    # set substage name, if it is not set
                    if not self.travis_substage.has_name() and \
                            self.travis_substage.has_command():
                        self.travis_substage.set_name(
                            self.get_substage_name(
                                self.travis_substage.get_command()
                            )
                        )

                    # only log complete substages
                    if not self.travis_substage.finished_incomplete:
                        self.current_job.add_stage(self.travis_substage.stage)
                    self.travis_substage = TravisSubstage()

    def parse_travis_worker_tag(self, line):
        """
        Parse and process Travis CI worker tag.

        Parameters:
        - line : line from logfile containing Travis CI tags
        """
        logger.debug('line : %s', line)

        # parse Travis CI worker tags
        result = re.search(TRAVIS_LOG_PARSE_WORKER_STRING, line)
        if not result:
            return

        worker_tags = result.groupdict()

        # check if parameter worker_tags is a dictionary and
        # if it contains all required tags
        tag_list = list({'hostname', 'os'})
        if tools.check_dict(worker_tags, "worker_tags", tag_list):
            logger.debug("Worker tags : %s", worker_tags)
            self.current_job.add_property("worker", worker_tags)

    def has_timing_tags(self):
        """
        Check if Travis CI job log has timing tags.

        Timing tags were introduced on Travis CI starting 2014-08-07,
        check if started_at is more recent.
        """
        started_at = self.current_job.get_property("started_at")
        if started_at is None or "timestamp_seconds" not in started_at:
            return False

        # 1407369600 is epoch timestamp of 2014-08-07T00:00:00Z
        return started_at["timestamp_seconds"] > 1407369600

    def get_job_duration(self):
        """Calculate build job duration."""
        started_at = self.current_job.get_property("started_at")
        finished_at = self.current_job.get_property("finished_at")
        if started_at is None or "timestamp_seconds" not in started_at or \
                finished_at is None or "timestamp_seconds" not in finished_at:
            return 0.0

        timestamp_start = float(started_at["timestamp_seconds"])
        timestamp_end = float(finished_at["timestamp_seconds"])
        return timestamp_end - timestamp_start

    def get_started_at(self):
        """Retrieve timestamp when build was started."""
        if tools.check_dict(self.current_build_data, key_list=["started_at"]):
            return self.current_build_data['started_at']
        else:
            return None

    def get_finished_at(self):
        """Retrieve timestamp when build finished."""
        if tools.check_dict(self.current_build_data, key_list=["finished_at"]):
            return self.current_build_data['finished_at']
        else:
            return None