#!python
#################################################
#===============================================#
# |                     Ascii Art :)        | # #
# |                                         | # #
# --------------------------------------------- #
# ---- Keysboard for Linux by Dunkmania101 ---- #
# --------------------------------------------- #
#################################################

import os, sys, subprocess, json
from json import JSONDecodeError
from threading import Thread
from time import sleep
from evdev import InputDevice, uinput, categorize, ecodes

##########
# Config:
universal_exit_key = "KEY_ESC" # Just in case the one in the config doesn"t work. Set to something that isn"t a keycode to disable.
indent_amount = 4 # Amount in spaces to indent saved Json.
shorten_name_amount = 15 # Amount in characters to shorten long names to for the log. Set to -1 to disable.

# Config Defaults:
config_file_def = os.path.expanduser("~/.config/keysboard/keysboard-conf.json") # Path to the default JSON config file. Can be overridden from the command line.
first_device_def = "the_path_to_a_device" # Default device path to use in newly generated configs.
device_nickname_def = "NotARealDevice" # Default nickname given to new devices. Nicknames are used as friendly names when printing.
exit_key_def = "KEY_ESC" # Default exit key in a newly added device.
first_layer_def = "main" # Default name for the first layer in a newly added device.
exit_cmd_default = "echo Keysboard has exited!" # Default command for each newly generated layer to run on exit.
hold_time_def = 0.1 # Default hold time for simulated keypresses (used if nothing is specified in the config).
##########

##########
# Start Script
##########
# Dict Tags/Values
# Action Types
action_set_layer_tag = "set_layer"
action_shell_tag = "shell"
action_keyboard_tag = "keyboard"
action_alias_tag = "alias"
action_multi_tag = "multi"

# Devices
devices_tag = "devices"
device_nickname_tag = "device_nickname"
aliases_tag = "aliases"
exit_key_tag = "exit_key"
exit_cmd_tag = "exit_cmd"

# Layers
layers_tag = "layers"
first_layer_tag = "first_layer"
keybinds_tag = "keybinds"
inherit_tag = "inherit"

# Printing
print_key_codes_tag = "print_key_codes"
print_actions_tag = "print_actions"

# Actions
action_type_tag = "action_type"
action_tag = "action"

# Key Simulation
keys_tag = "keys"
key_tag = "key"
hold_time_tag = "hold_time"
delay_tag = "delay"
set_key_tag = "set_key"

# Other
backup_tag = "backup"
##########

##########
def gen_log_msg(device="", current_layer="", msg="", exclude_empty=True):
    if exclude_empty and device == "":
        device_msg = ""
    else:
        device_msg = f" Device = [ {device} ]"
    if exclude_empty and current_layer == "":
        layer_msg = ""
    else:
        layer_msg = f" / Layer = [ {current_layer} ]"

    if exclude_empty and device_msg == "" and layer_msg == "":
        device_layer_msg = ""
    else:
        device_layer_msg = f"[ {device_msg}{layer_msg} ]: "

    return f"{device_layer_msg}{msg}"

def exec_cmd(cmd, device="", current_layer="", print_output=True):
    output = str(subprocess.run(cmd, shell=True, start_new_session=True, stdout=subprocess.PIPE).stdout.decode("utf-8"))
    if print_output and output != "" and output != "\n":
        if "\n" in output:
            output = "|".join(output.splitlines())
        print(gen_log_msg(device, current_layer, f"Command output: [ {output} ]"))

def run_thread(target, args):
    Thread(target=target, args=args, daemon=False).start()

def set_key(code, state=0, dev=uinput.UInput()):
    key = ecodes.ecodes[code]
    dev.write(ecodes.EV_KEY, key, state)
    dev.syn()

def press_key(code, hold_time=hold_time_def, dev=uinput.UInput()):
    set_key(code, 1, dev)
    sleep(hold_time)
    set_key(code, 0, dev)

def press_keys(keys, dev=uinput.UInput()):
    for key in keys:
        if set_key_tag in key.keys():
            set_key(key[key_tag], key[set_key_tag], dev)
        else:
            if delay_tag in key.keys():
                sleep(key[delay_tag])
            press_key(key[key_tag], key.get(hold_time_tag, hold_time_def), dev)

def blank_layer():
    return {
        keybinds_tag: {
            "KEY_1": {
                action_type_tag: action_set_layer_tag,
                action_tag: first_layer_def
            },
            "KEY_2": {
                action_tag: action_shell_tag,
                action_tag: "echo hi"
            }
        }
    }

def blank_device():
    return {
        device_nickname_tag: device_nickname_def,
        exit_cmd_tag: exit_cmd_default,
        exit_key_tag: exit_key_def,
        layers_tag: {
            first_layer_def: blank_layer()
        }
    }

def blank_config():
    return {
        devices_tag: {
            first_device_def: blank_device()
        }
    }

def mkdir_p(directory):
    try:
        os.makedirs(directory)
    except:
        pass

def open_config(overwrite, config_file):
    if not os.path.isfile(config_file):
        mkdir_p(os.path.dirname(config_file))
        f = open(config_file, "x")
        try:
            json.dump(blank_config(), f, indent=indent_amount)
        finally:
            f.close()
    if overwrite:
        return open(config_file, "w+")
    return open(config_file, "r+")

def save_config(config, config_file):
    try:
        f = open_config(True, config_file)
        json.dump(config, f, indent=indent_amount)
    finally:
        f.close()

def read_config(config_file):
    f = open_config(False, config_file)
    try:
        try:
            read_conf = f.read()
            conf = json.loads(read_conf)
        except JSONDecodeError:
            blank = blank_config()
            blank[backup_tag] = read_conf
            save_config(blank, config_file)
            try:
                read_conf = f.read()
                conf = json.loads(read_conf)
            except JSONDecodeError:
                conf = blank_config()
        return conf
    finally:
        f.close()

def add_device(device, config_file):
    config = read_config(config_file)

    if devices_tag not in config.keys():
        config[devices_tag] = {}
        save_config(config, config_file)

    if config.get(devices_tag, {}).get(device, {}) == {}:
        config[devices_tag][device] = blank_device()
        save_config(config, config_file)
    return config

def add_layer(device, layer, config_file):
    config = add_device(device, config_file)

    if layers_tag not in config[devices_tag][device].keys():
        config[devices_tag][device][layers_tag] = {}
        save_config(config, config_file)

    if config.get(devices_tag, {}).get(device, {}).get(layers_tag, {}).get(layer, {}) == {}:
        config[devices_tag][device][layers_tag][layer] = blank_layer()
        save_config(config, config_file)
    return config

def add_keybind(device, layer, key, action_type, action, config_file):
    config = add_layer(device, layer, config_file)

    if keybinds_tag not in config[devices_tag][device][layers_tag][layer].keys():
        config[devices_tag][device][layers_tag][layer][keybinds_tag] = {}
        save_config(config, config_file)

    if config.get(devices_tag, {}).get(device, {}).get(layers_tag, {}).get(layer, {}).get(keybinds_tag, {}).get(key, {}) == {}:
        config[devices_tag][device][layers_tag][layer][keybinds_tag][key] = {
            action_type_tag: action_type,
            action_tag: action
        }
        save_config(config, config_file)
    return config

def get_first_layer(device_config):
    layers = list(device_config[layers_tag].keys())
    if len(layers) > 0:
        return layers[0]
    return ""

class StringContainer():
    string = ""
    def set_string(self, string):
        if isinstance(string, str):
            self.string = string
    def get_string(self):
        return self.string
    def __eq__(self, other):
        return (isinstance(other, (str, StringContainer)) and other == self.get_string()) or super.__eq__(self, other)
    def __str__(self):
        return self.get_string()

def run_device(device, config_file):
    device_short = device
    if shorten_name_amount != -1 and len(device_short) > shorten_name_amount:
        device_short = device_short[:shorten_name_amount] + "..."
    dev_no_config_msg = f"Device [ {device_short} ] is not in the configuration, skipping..."
    invalid_dev_msg = f"Device [ {device_short} ] is invalid, skipping..."
    error_msg = gen_log_msg(device_short, "", f"Hit a critical error, stopping...")
    config = read_config(config_file)
    devices = config.get(devices_tag, {})
    if device in devices.keys():
        device_short = devices[device].get(device_nickname_tag, device_short)
        try:
            dev = InputDevice(device)
            try:
                dev.grab()
                device_config = devices[device]
                current_layer = StringContainer()
                def set_layer_default():
                    current_layer.set_string(device_config.get(first_layer_tag, get_first_layer(device_config)))
                set_layer_default()
                fake_dev = uinput.UInput()
                for event in dev.read_loop():
                    if event.type == ecodes.EV_KEY:
                        key = categorize(event)
                        if key.keystate == key.key_down:
                            config = read_config(config_file)
                            devices = config.get(devices_tag, {})
                            if device in devices:
                                device_config = devices[device]
                                device_aliases = device_config.get(aliases_tag, {})
                                device_short = device_config.get(device_nickname_tag, device_short)
                                code = key.keycode
                                if print_key_codes_tag in device_config.keys():
                                    print_key_codes = device_config[print_key_codes_tag]
                                else:
                                    print_key_codes = True
                                if print_key_codes:
                                    print(gen_log_msg(device_short, current_layer.get_string(), f"Pressed key: [ {code} ]"))
                                exit_key = device_config.get(exit_key_tag, None)
                                if code == exit_key or code == universal_exit_key:
                                    print(gen_log_msg(device_short, current_layer.get_string(), "Exit key pressed! Quitting..."))
                                    if exit_cmd_tag in device_config:
                                        exit_cmd = device_config[exit_cmd_tag]
                                        if isinstance(exit_cmd, str):
                                            try:
                                                run_thread(exec_cmd, (exit_cmd,))
                                            except:
                                                print(gen_log_msg(device_short, current_layer.get_string(), f"Device [ {device_short} ] hit an error while running its exit command [ {exit_cmd} ], ignoring..."))
                                    break
                                else:
                                    layers = device_config.get(layers_tag, {})
                                    if current_layer.get_string() in layers:
                                        keybinds = {}
                                        inherited_layers = layers[current_layer.get_string()].get(inherit_tag, [])
                                        if isinstance(inherited_layers, str) and inherited_layers in layers.keys():
                                            inherited_layers = layers[inherited_layers]
                                        if isinstance(inherited_layers, list):
                                            for inherited_layer in inherited_layers:
                                                if inherited_layer in layers:
                                                    keybinds.update(layers[inherited_layer].get(keybinds_tag, {}))
                                        keybinds.update(layers[current_layer.get_string()].get(keybinds_tag, {}))
                                        for key in keybinds.keys():
                                            if (isinstance(code, list) and key in code) or (isinstance(code, str) and code == key):
                                                old_aliases = []
                                                def exec_action(bind):
                                                    if action_tag in bind.keys():
                                                        action = bind[action_tag]
                                                        action_type = ""
                                                        if print_actions_tag in device_config:
                                                            print_actions = device_config[print_actions_tag]
                                                        else:
                                                            print_actions = True
                                                        if action_type_tag in bind:
                                                            action_type = bind[action_type_tag]
                                                        if action_type == action_shell_tag or action_type == "":
                                                            if print_actions:
                                                                print(gen_log_msg(device_short, current_layer.get_string(), f"Running in system shell: [ {action} ]"))
                                                            try:
                                                                run_thread(exec_cmd, (action, device_short, current_layer.get_string(), print_actions,))
                                                            except:
                                                                print(gen_log_msg(device_short, current_layer.get_string(), f"Hit an error while running shell command [ {action} ], ignoring..."))
                                                        elif action_type == action_keyboard_tag:
                                                            if type(action) is list:
                                                                run_thread(press_keys, (action, fake_dev,))
                                                            elif key_tag in action.keys():
                                                                code = action[key_tag]
                                                                if set_key_tag in action.keys():
                                                                    run_thread(set_key, (code, bind[set_key_tag], fake_dev,))
                                                                else:
                                                                    run_thread(press_key, (code, bind.get(hold_time_tag, hold_time_def,), fake_dev))
                                                        elif action_type == action_set_layer_tag:
                                                            if action in layers.keys():
                                                                if print_actions:
                                                                    print(gen_log_msg(device_short, "", f"Switching to layer: [ {action} ], From Layer: [ {current_layer.get_string()} ]"))
                                                                current_layer.set_string(action)
                                                            else:
                                                                print(gen_log_msg(device_short, current_layer.get_string(), f"Layer [ {action} ] is not in config, ignoring..."))
                                                        elif action_type == action_alias_tag:
                                                            if action in device_aliases.keys():
                                                                if action in old_aliases:
                                                                    print(gen_log_msg(device_short, current_layer.get_string(), f"Action alias [ {action} ] has been recursively called, skipping..."))
                                                                else:
                                                                    old_aliases.append(action)
                                                                    exec_action(device_aliases[action])
                                                            else:
                                                                print(gen_log_msg(device_short, current_layer.get_string(), f"Action alias [ {action} ] is undefined, skipping..."))
                                                        elif action_type == action_multi_tag:
                                                            if isinstance(action, list):
                                                                for sub_action in action:
                                                                    exec_action(sub_action)
                                                                    if isinstance(sub_action, dict):
                                                                        sub_action_type = sub_action.get(action_type_tag, "")
                                                                        if sub_action_type == action_alias_tag and action_tag in sub_action and sub_action[action_tag] in old_aliases:
                                                                            old_aliases.remove(sub_action[action_tag])
                                                            else:
                                                                print(gen_log_msg(device, current_layer.get_string(), f"Multi-Action [ {action} ] is not of type list, skipping..."))
                                                exec_action(keybinds[key])
                                    else:
                                        set_layer_default()
                            else:
                                print(dev_no_config_msg)
                                break
            except IOError:
                print(gen_log_msg(device_short, "", "Already grabbed, skipping..."))
                pass
            except Exception as error:
                print(f"{error_msg}: {str(error)}")
                pass
            finally:
                try:
                    dev.ungrab()
                except IOError:
                    pass
                fake_dev.close()
        except IOError:
            print(invalid_dev_msg)
            pass
    else:
        print(dev_no_config_msg)

def run_devices(config_file):
    for device in read_config(config_file).get(devices_tag, {}).keys():
        try:
            run_thread(run_device, (device, config_file,))
        except Exception as error:
            print(gen_log_msg(device, "", f"Failed to start, skipping...: {str(error)}"))
            pass

def main_run(args):
    config_file = config_file_def
    if len(args) > 1:
        invalid_cmdline_msg = "Invalid command line arguments!"
        usage_msg = """
Usage:

Run normally: python keysboard.py
Add device: python keysboard.py add-device device_name
Add layer to device: python keysboard.py add-device device_name layer_name
Add keybind to layer of device: python keysboard.py add-keybind device_name layer_name keycode action_type \"action\"

Run with different config: python keysboard.py command options config=path/to/config.json

Valid action_type values: shell, set_layer
Valid action values (respectively): any shell command, any layer of same device

Requires python 3!
        """
        def print_usage_msg(invalid_cmdline=False):
            if invalid_cmdline:
                print(invalid_cmdline_msg)
            print(usage_msg)
        config_arg = "config="
        for arg in args:
            if arg.startswith(config_arg):
                config_file = os.path.expanduser(arg.replace(config_arg, ""))
                args.remove(arg)
        if len(args) > 1:
            if "help" in args or "h" in args:
                print(usage_msg)
            elif "add-device" in args:
                if len(args) >= 3:
                    add_device(args[2])
                else:
                    print_usage_msg(True)
            elif "add-layer" in args:
                if len(args) >= 4:
                    add_layer(args[2], args[3])
                else:
                    print_usage_msg(True)
            elif "add-keybind" in args:
                if len(args) >= 7:
                    add_keybind(args[2], args[3], args[4], args[5], args[6])
                else:
                    print_usage_msg(True)
            else:
                print_usage_msg(True)
        else:
            run_devices(config_file)
    else:
        run_devices(config_file)

if __name__ == "__main__":
    main_run(sys.argv)


# coding: utf-8
from openerp import models, fields

class TagModel(models.Model):
    _name = 'money.tag'

    name = fields.Char(required=True)

    _sql_constraints = [
        ('name_uniq', 'unique(name)', 'Tag name must be unique!'),
    ]


from caler import *
import pandas as pd
from Plotor import *
from Reader import *

no_today = {'IF', 'IH', 'IC'}

def GetCon(ticker):
  for i, c in enumerate(ticker):
    if c.isdigit():
      return ticker[:i]

class Trader:
  def __init__(self, enable_fee = False, fee_rate=0.0, record=False, prefix='', show_raw = True):
    self.caler = CALER('/root/hft/config/contract/bk_contract.config')
    self.pt = Plotor()
    self.pos = {}
    self.avgcost = {}
    self.pnl = {}
    self.raw_pnl = {}
    self.pnl_hist = {}
    self.raw_pnl_hist = {}
    self.fee = {}
    self.trade_round = {}
    self.fee_rate = fee_rate
    self.record = record
    self.prefix = prefix
    self.show_raw = show_raw
    if record == True:
      self.f = open('traders_record.txt', 'w')

  def __def__(self):
    if self.record:
      self.f.close()

  def RegisterOneTrade(self, ticker, size, price):
    assert price > 0
    assert isinstance(size, int)
    assert isinstance(ticker, str)
    if ticker not in self.pos:
      self.pos[ticker] = size
      self.avgcost[ticker] = price
      self.pnl[ticker] = 0.0
      self.raw_pnl[ticker] = 0.0
      self.trade_round[ticker] = 1.0
      self.fee[ticker] = 0.0
      self.pnl_hist[ticker] = []
      self.raw_pnl_hist[ticker] = []
      return
    if self.pos[ticker] * size < 0:  # close position
      if abs(size) > self.pos[ticker]:  # over close
        c = self.caler.CalFee(ticker, self.avgcost[ticker], abs(self.pos[ticker]), price, abs(self.pos[ticker]), GetCon(ticker) in no_today)
        self.fee[ticker] += c.open_fee + c.close_fee
        self.pnl[ticker] += self.caler.CalNetPnl(ticker, self.avgcost[ticker], abs(self.pos[ticker]), price, abs(self.pos[ticker]), OrderSide.Buy if size > 0 else OrderSide.Sell, GetCon(ticker) in no_today)
        self.raw_pnl[ticker] += self.caler.CalPnl(ticker, self.avgcost[ticker], abs(self.pos[ticker]), price, abs(self.pos[ticker]), OrderSide.Buy if size > 0 else OrderSide.Sell)
        self.avgcost[ticker] = price
      else: # normal close
        c = self.caler.CalFee(ticker, self.avgcost[ticker], abs(size), price, abs(size), GetCon(ticker) in no_today)
        self.fee[ticker] += c.open_fee + c.close_fee
        self.pnl[ticker] += self.caler.CalNetPnl(ticker, self.avgcost[ticker], abs(size), price, abs(size), OrderSide.Buy if size > 0 else OrderSide.Sell, GetCon(ticker) in no_today)
        self.raw_pnl[ticker] += self.caler.CalPnl(ticker, self.avgcost[ticker], abs(size), price, abs(size), OrderSide.Buy if size > 0 else OrderSide.Sell)
      self.pnl_hist[ticker].append(self.pnl[ticker])
      self.raw_pnl_hist[ticker].append(self.raw_pnl[ticker])
    else:  # open position
      self.avgcost[ticker] += (price-self.avgcost[ticker])*size/(self.pos[ticker]+size)
    self.pos[ticker] += size
    if self.pos[ticker] == 0:
      self.avgcost[ticker] = 0.0
    self.trade_round[ticker] += 1
    trade_record = "Trade %s %d@%f" %(ticker, size, price)
    if self.record:
      self.f.write(trade_record+'\n')

  def PNL(self):
    print(self.pnl_hist)
    print(self.raw_pnl_hist)
    print(self.GetStratPnl())

  def GetTickerPnl(self, prefix=''):
    return {i:{prefix+'raw':[0.0]+self.raw_pnl_hist[i], prefix+'net': [0.0]+self.pnl_hist[i]} for i in self.pnl_hist} if self.show_raw else {i:{prefix+'net': [0.0]+self.pnl_hist[i]} for i in self.pnl_hist}

  def GetStratPnl(self, prefix=''):
    self.strat_pnl_hist = {}
    self.raw_strat_pnl_hist = {}
    m = self.GetTickerPnl(prefix)
    for t in m:
      con = GetCon(t)
      if con == None:
        continue
      if con not in self.strat_pnl_hist:
        self.strat_pnl_hist[con] = self.pnl_hist[t]
        self.raw_strat_pnl_hist[con] = self.raw_pnl_hist[t]
        continue
      if len(self.pnl_hist[t]) != len(self.strat_pnl_hist[con]):
        print('main and hedge ticker pnl list length is not equal %d %d' %(len(self.pnl_hist[t]), len(self.strat_pnl_hist[con])))
      for i, c in enumerate(self.strat_pnl_hist[con]):
        #print(self.strat_pnl_hist[con])
        #print(self.pnl_hist[t])
        #print(len(self.strat_pnl_hist[con]))
        if len(self.strat_pnl_hist[con]) != len(self.pnl_hist[t]):
          break
        #print(len(self.pnl_hist[t]))
        self.strat_pnl_hist[con][i] += self.pnl_hist[t][i]
        self.raw_strat_pnl_hist[con][i] += self.raw_pnl_hist[t][i]
    return {i:{prefix+'raw':[0.0]+self.raw_strat_pnl_hist[i], prefix+'net': [0.0]+self.strat_pnl_hist[i]} for i in self.strat_pnl_hist} if self.show_raw else {i:{prefix+'net': [0.0]+self.strat_pnl_hist[i]} for i in self.strat_pnl_hist}

  def PlotStratPnl(self, file_name='strat_pnl_hist', show=False):
    self.pt.MultiPlot(self.GetStratPnl(), file_name, show=show, prefix=self.prefix)

  def Summary(self):
    print(self.GenDFReport())
    print(self.GenStratReport())

  def HistorySummary(self):
    print('================================================================================================================')
    sort_key = sorted(self.pos.keys())
    for t in sort_key:
      #print('for %s trade_round=%d, pnl=%.2f, left_pos=%.2f, avgcost=%.2f, for feerate %f, rough_fee is %.2f' %(t, self.trade_round[t], self.pnl[t], self.pos[t], self.avgcost[t], self.fee_rate, abs(self.fee_rate*self.trade_round[t]*self.avgcost[t])))
      print('for %s trade_round=%d, pnl=%.2f, left_pos=%.2f, avgcost=%.2f, rough_fee is %.2f' %(t, self.trade_round[t], self.pnl[t], self.pos[t], self.avgcost[t], self.fee[t]))
    print('===============================================================================================================')

  def GenDFReport(self):
    r = []
    sort_key = sorted(self.pos.keys())
    for t in sort_key:
      pnl_l = self.pnl_hist[t]
      r.append([t, self.trade_round[t], round(self.pnl[t],1), self.pos[t], self.avgcost[t], round(self.fee[t],1), np.mean(pnl_l)/(np.std(pnl_l)+0.01)])
    df = pd.DataFrame(r, columns = ['ticker', 'trade_round', 'net_pnl', 'left_position', 'avgcost', 'fee', 'sharp'])
    df = df.set_index('ticker')
    return df

  def GenStratReport(self, df=''):
    if len(df) == 0:
      df = self.GenDFReport()
    ticker_strat = {}
    for k in df.index:
      ticker_strat[k] = GetCon(k)
    m = df.T.to_dict()
    self.strat_pnl = {}
    for k in m:
      con = GetCon(k)
      if con not in self.strat_pnl:
        self.strat_pnl[con] = m[k]
        continue
      self.strat_pnl[con]['net_pnl'] += m[k]['net_pnl']
      self.strat_pnl[con]['trade_round'] = max(m[k]['trade_round'], self.strat_pnl[con]['trade_round'])
      self.strat_pnl[con]['left_position'] = max(m[k]['left_position'], self.strat_pnl[con]['left_position'])
      self.strat_pnl[con]['avgcost'] -= m[k]['avgcost']
      self.strat_pnl[con]['avgcost'] = -self.strat_pnl[con]['avgcost']
      self.strat_pnl[con]['fee'] += m[k]['fee']
    for k in self.strat_pnl:
      for c in self.strat_pnl[k].keys():
        if isinstance(self.strat_pnl[k][c], float):
          self.strat_pnl[k][c] = round(self.strat_pnl[k][c], 1)
    rdf = pd.DataFrame(self.strat_pnl).T
    return rdf[['trade_round', 'net_pnl', 'left_position', 'avgcost', 'fee']]

if __name__=='__main__':
  t = Trader()
  r = Reader()
  #r.load_order_file('/root/hft/build/bin/order.dat')
  r.load_order_file('/root/hft/build/bin/order_2020-03-17.dat')
  s = r.get_ordersize()
  count = 0
  for i in range(s):
    o = r.read_border(i)
    if o.price > 0 and o.size > 0:
      #o.Show()
      o.ticker= GetCon(o.ticker) + ('8888' if count%2 == 0 else '9999')
      t.RegisterOneTrade(o.ticker, o.size if o.side == 1 else -o.size, o.price)
      #t.RegisterOneTrade(o.ticker, o.size, o.price)
      count += 1
  #t.RegisterOneTrade("IC8888", -3, 5800)
  #t.RegisterOneTrade("IC9999", -3, 5800)
  #t.RegisterOneTrade("IC8888", -3, 5800)
  #t.RegisterOneTrade("IC9999", -3, 5800)
  t.Summary()
  t.PlotStratPnl(show=True)
  #print(t.GenStratReport())
  #print(t.GenDFReport())


class Solution:
    def thousandSeparator(self, n: int) -> str:
        if n < 1000: return str(n)
        ans = []
        n = str(n)
        len_t = len(n)

        for i in range(len(n) - 1, -1, -1):
            idx = len_t - i
            ans.insert(0, n[i])
            if idx != len_t and idx % 3 == 0:
                ans.insert(0, '.')

        return ''.join(ans)


# -*- coding:utf-8 -*-

from torch import nn
from torchvision import models
import torch

class Net(nn.Module):
    def __init__(self):
        """Declare all needed layers."""
        super(Net, self).__init__()
        resnet = models.resnet50()
        self.features = nn.Sequential(*list(resnet.children())[:-1])
        self.fc = nn.Sequential(
            nn.Linear(2048, 1024),
            nn.Dropout(0.4),
            nn.LeakyReLU(),
            nn.Linear(1024, 256),
            nn.Dropout(0.4),
            nn.LeakyReLU(),
        )

        self.fc_s = nn.Sequential(nn.Linear(256, 1),
                                  nn.LeakyReLU())

    def forward(self, X):
        X1, X2 = X.chunk(2, 1)
        x1 = X1.contiguous().view(
            X1.size()[0] * X1.size()[1], X1.size()[2], X1.size()[3], X1.size()[4]
        )
        x2 = X2.contiguous().view(
            X2.size()[0] * X2.size()[1], X2.size()[2], X2.size()[3], X2.size()[4]
        )

        x1 = self.features(x1)
        x1 = x1.view(x1.size(0), -1)
        x1 = self.fc(x1)
        x1_s = self.fc_s(x1)

        x2 = self.features(x2)
        x2 = x2.view(x2.size(0), -1)
        x2 = self.fc(x2)
        x2_s = self.fc_s(x2)

        x1_s = x1_s.view(-1, 3).mean(dim=1)
        x2_s = x2_s.view(-1, 3).mean(dim=1)
        s = torch.stack((x1_s, x2_s), dim=1).mean(dim=1)
        return s


#!/usr/bin/python3

from selenium import webdriver
import time

driver = webdriver.Firefox()
driver.get('http://m.mail.10086.cn')

driver.set_window_size(480, 800)

time.sleep(10)
driver.quit()


__author__ = 'Iamokthanks'

import iSchedule

if __name__ == '__main__':
    print("Hello World")
    sched = iSchedule.iSchedule()
    sched.initTask()

"""
from apscheduler.scheduler import Scheduler
import time

sched = Scheduler()
sched.start()

def job_fucntion():
    print("Hello.Python")

def job_2():
    print("Hello.Perl")

sched.add_cron_job(job_fucntion, second='*', hour='*', day='*')
sched.add_cron_job(job_2, second='*/10', hour='*', day='*')

while True:
    time.sleep(1)
"""


import sqlite3

DB_PATH = '/Users/vishalnalwa/projects/project_python/db'
DB_NAME = 'music.sqlite'
conn = sqlite3.connect(DB_PATH+'/'+DB_NAME)
cur = conn.cursor()

cur.execute('DROP TABLE IF EXISTS Tracks')
cur.execute('CREATE TABLE Tracks (title TEXT , plays INTEGER)')

conn.close()


#  Hint:  You may not need all of these.  Remove the unused functions.
from hashtables import (HashTable,
                        hash_table_insert,
                        hash_table_remove,
                        hash_table_retrieve,
                        hash_table_resize)

def get_indices_of_item_weights(weights, length, limit):

    ht = HashTable(16)
    ht2 = HashTable(16)

    """
    YOUR CODE HERE
    """
    if limit == None:
        return None
    # Declare count property
    count = 0
    # Put array elements into two hash tables, using for loop,
    for i in range(0, len(weights)):
        hash_table_insert(ht, count, weights[i])
        count += 1

    # If the count is one, return None
    if count == 1 or count == 0:
        return None

    # Declare answer property
    answer = (1, 0)
    # Declare two index properties
    left_index = 0
    right_index = 1
    # While the sum of the two elements of answer do not equal the wait limit
    while hash_table_retrieve(ht, answer[0]) + hash_table_retrieve(ht, answer[1]) != limit:
        # answer equals (left index, right index)
        if hash_table_retrieve(ht, left_index) != None and hash_table_retrieve(ht, right_index) != None:

            answer = (left_index, right_index)
        # If the left index is at the last index already, then break out of the loop here to avoid index out of range errors
        if left_index == count - 1:
            break
        # If the right index is at the last index, then the left index is increased by one and the right index is reset to 0 to start the loop again
        if right_index >= count - 1:
            left_index += 1
            right_index = left_index + 1
        right_index += 1
    # If we are out of the loop but the answer values do not add up to the limit, set the answer to none
    if hash_table_retrieve(ht, answer[0]) + hash_table_retrieve(ht, answer[1]) != limit:
        return None
    # Else if the second element of answer is greater than the first, swap element indexes
    elif answer[1] > answer[0]:
        answer = (answer[1], answer[0])
    # Return answer
    return answer

def print_answer(answer):
    if answer is None:
        print(str(answer[0] + " " + answer[1]))
    else:
        print("None")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

#+#FIDASIM Utilities
#+This file contains useful FIDASIM utilities
#+***
from __future__ import print_function
import os
from os.path import dirname
import subprocess
import platform
import numpy as np
import h5py

def get_fidasim_dir():
    """
    #+#get_fidasim_dir
    #+ Gets FIDASIM install directory
    #+***
    #+##Output Arguments
    #+     **directory**: FIDASIM install directory.
    #+##Example Usage
    #+```python
    #+>>> fida_dir = get_fidasim_dir()
    #+```
    """

    directory = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))

    return directory

def get_version(fidasim_dir):
    """
    #+#get_version
    #+ Gets FIDASIM version number from git.
    #+ Falls back to reading VERSION file when git is not available
    #+***
    #+##Input Arguments
    #+    **fidasim_dir**: FIDASIM install directory
    #+
    #+##Output Arguments
    #+     **version**: FIDAIM version number.
    #+
    #+##Example Usage
    #+```python
    #+>>> version = get_version(get_fidasim_dir())
    #+```
    """
    version = ''
    alt = False

    if platform.system() == 'Windows':
        alt = True
    else:
        # Location of .git folder
        git_dir = r'{}{}.git'.format(fidasim_dir, os.path.sep)

        # git is installed if git_file is a file
        proc = subprocess.Popen('command -v git', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        git_file = proc.communicate()[0].decode('utf-8')
        git_file = git_file.replace('\n', '')

        # Check that .git folder is present and git is installed
        if os.path.isfile(git_file) and os.path.isdir(git_dir):
            try:
                version = subprocess.check_output(['git', '--git-dir={}'.format(git_dir), 'describe', '--tags', '--always', '--dirty'])
                version = version.replace('\n', '')
            except:
                alt = True
        else:
            alt = True

    # If above didn't work, read version file
    if alt:
        # Git 'version' filepath
        ver_file = '{}{}VERSION'.format(fidasim_dir, os.path.sep)

        if os.path.isfile(ver_file):
            with open(ver_file) as f:
                version = f.read()

    return version

def rz_grid(rmin, rmax, nr, zmin, zmax, nz):
    """
    #+#rz_grid
    #+Creates interpolation grid
    #+***
    #+##Arguments
    #+    **rmin**: Minimum radius [cm]
    #+
    #+    **rmax**: Maximum radius [cm]
    #+
    #+    **nr**: Number of radii
    #+
    #+    **zmin**: Minimum Z value [cm]
    #+
    #+    **zmax**: Maximum Z value [cm]
    #+
    #+    **nz**: Number of z values
    #+
    #+##Return Value
    #+Interpolation grid dictionary
    #+
    #+##Example Usage
    #+```python
    #+>>> grid = rz_grid(0,200.0,200,-100,100,200)
    #+```
    """
    dr = (rmax - rmin) / (nr - 1)
    dz = (zmax - zmin) / (nz - 1)
    r = rmin + dr * np.arange(nr, dtype=np.float64)
    z = zmin + dz * np.arange(nz, dtype=np.float64)

    r2d = np.tile(r, (nz, 1)).T
    z2d = np.tile(z, (nr, 1))

    grid = {'r2d': r2d,
            'z2d': z2d,
            'r': r,
            'z': z,
            'nr': nr,
            'nz': nz}

    return grid

def colored(text, color): #, on_color=None, attrs=None):
    """
    #+#colored
    #+ Return text string formatting for color in terminal
    #+***
    #+##Input Arguments
    #+     **text**: String to be colored
    #+
    #+     **color**: Desired color of string. Red, green, yellow, blue, magenta, cyan, or white.
    #+
    #+##Output Arguments
    #+     **text**: Text formated to have "color" in terminal.
    #+##Example Usage
    #+```python
    #+>>> text = colored("Text to be red", 'red')
    #+>>> print(text)
    #+```
    """
    # Copyright (c) 2008-2011 Volvox Development Team
    #
    # Permission is hereby granted, free of charge, to any person obtaining a copy
    # of this software and associated documentation files (the "Software"), to deal
    # in the Software without restriction, including without limitation the rights
    # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    # copies of the Software, and to permit persons to whom the Software is
    # furnished to do so, subject to the following conditions:
    #
    # The above copyright notice and this permission notice shall be included in
    # all copies or substantial portions of the Software.
    #
    # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    # THE SOFTWARE.
    #
    # Author: Konstantin Lepa <konstantin.lepa@gmail.com>
    COLORS = dict(list(zip(['grey',
                            'red',
                            'green',
                            'yellow',
                            'blue',
                            'magenta',
                            'cyan',
                            'white',],
                            list(range(30, 38)))))

    RESET = '\033[0m'

    if os.getenv('ANSI_COLORS_DISABLED') is None:
        fmt_str = '\033[%dm%s'

        text = fmt_str % (COLORS[color], text)

        text += RESET

    return text

def info(string):
    """
    #+#info
    #+Print a informational message
    #+***
    #+##Arguments
    #+     **str**: message
    #+
    #+##Example Usage
    #+```python
    #+>>> info("This is an informative message")
    #+```
    """
    print(colored('INFO: ' + string, 'cyan'))

def warn(string):
    """
    #+#warn
    #+Print a warning message
    #+***
    #+##Arguments
    #+     **string**: message
    #+
    #+##Example Usage
    #+```python
    #+>>> warn("This may be a problem")
    #+```
    """
    print(colored('WARNING: ' + string, 'magenta'))

def error(string, halt=False):
    """
    #+#error
    #+Print a error message
    #+***
    #+##Arguments
    #+     **string**: message
    #+
    #+##Keyword Arguments
    #+     **halt**: Halt program execution
    #+
    #+##Example Usage
    #+```python
    #+>>> error("Error message")
    #+```
    """
    print(colored('ERROR: {}'.format(string), 'red'))

    if halt:
        raise Exception()

def success(string):
    """
    #+#success
    #+Print a success message
    #+***
    #+##Arguments
    #+     **string**: message
    #+
    #+##Example Usage
    #+```python
    #+>>> success("Yay!!!")
    #+```
    """
    print(colored('SUCCESS: ' + string, 'green'))

def beam_grid(nbi, rstart,
              nx=None, ny=None, nz=None, dv=8.0,
              length=100.0, width=80.0, height=80.0):
    """
    #+#beam_grid
    #+ Calculates settings for a grid that aligns with the neutral beam.
    #+***
    #+##Arguments
    #+    **nbi**: [Neutral beam geometry structure](|url|/page/03_technical/01_prefida_inputs.html#neutral-beam-geometry-structure)
    #+
    #+    **rstart**: Radial start position of beam grid [cm]
    #+
    #+##Keyword Arguments
    #+    **dV**: Cell volume [\(cm^3\)]: Defaults to 8.0
    #+
    #+    **nx**: Number of cells in length: Default determined by `dV`
    #+
    #+    **ny**: Number of cells in width: Default determined by `dV`
    #+
    #+    **nz**: Number of cells in height: Default determined by `dV`
    #+
    #+    **length**: Length of grid along beam sightline. [cm]: Defaults to 100 cm
    #+
    #+    **width**: Width of grid [cm]: Defaults to 100 cm
    #+
    #+    **height**: Height of grid [cm]: Defaults 80 cm
    #+
    #+##Return Value
    #+    Structure containing beam grid settings suitable for the Namelist File
    #+
    #+##Example Usage
    #+```python
    #+>>> grid = beam_grid(nbi,200.0,nx=100,ny=50,nz=50,length=100,width=50,height=50)
    #+```
    """

    if width < nbi['widy']:
        warn("Grid width is smaller then the source width")
        print("width: {}".format(width))
        print("source width: {}".format(nbi['widy']))

    if height < nbi['widz']:
        warn("Grid height is smaller then the source height")
        print("height: {}".format(height))
        print("source height: {}".format(nbi['widz']))

    dv3 = dv ** (1. / 3.)

    if nx is None:
        nx = round(length / dv3)

    if ny is None:
        ny = round(width / dv3)

    if nz is None:
        nz = round(height / dv3)

    xmin = 0.
    xmax = length
    ymin = -width / 2.
    ymax = width / 2.
    zmin = -height / 2.
    zmax = height / 2.

    src = nbi['src']
    axis = nbi['axis'] / np.sqrt(np.sum(nbi['axis'] ** 2))
    pos = src + 100. * axis

    if np.sqrt(src[0] ** 2 + src[1] ** 2) < rstart:
        error("Source radius cannot be less then rstart", halt=True)

    dis = np.sqrt(np.sum((src - pos) ** 2.0))
    beta = np.arcsin((src[2] - pos[2]) / dis)
    alpha = np.arctan2((pos[1] - src[1]), (pos[0] - src[0]))
    gamma = 0.
    a = axis[0] ** 2 + axis[1] ** 2
    b = 2. * (src[0] * axis[0] + src[1] * axis[1])
    c = src[0] ** 2 + src[1] ** 2 - rstart ** 2
    t = (-b - np.sqrt(b ** 2 - 4. * a * c)) / (2. * a)
    origin = src + t * axis

    beam_grid = {'nx': nx,
                 'ny':  ny,
                 'nz':  nz,
                 'xmin': xmin,
                 'xmax': xmax,
                 'ymin': ymin,
                 'ymax': ymax,
                 'zmin': zmin,
                 'zmax': zmax,
                 'alpha': alpha,
                 'beta': beta,
                 'gamma': gamma,
                 'origin': origin}

    return beam_grid

def write_data(h5_obj, dic, desc, units, name=''):
    """
    #+#write_data
    #+ Write h5 datasets with attributes 'description' and 'units'
    #+***
    #+##Arguments
    #+     **h5_obj**: An h5 file or group object from h5py
    #+
    #+     **dic**: Dict of data to save as h5 datasets
    #+
    #+     **desc**: Dict with same keys as dic describing each item in dic
    #+
    #+     **units**: Dict with same keys as dic providing units of data in dic, doesn't have to be all keys of dic.
    #+
    #+##Keyword Arguments
    #+     **name**: Name/description of dic for clarity in raising errors
    #+
    #+##Example Usage
    #+```python
    #+>>> write_data(h5_obj, dic, desc, units)
    #+```
    """
    for key in dic:
        # Transpose data to match expected by Fortran and historically provided by IDL
        if isinstance(dic[key], np.ndarray):
            if dic[key].ndim >= 2:
                dic[key] = dic[key].T

        # Make strings of fixed length as required by Fortran.
        # See http://docs.h5py.org/en/latest/strings.html#fixed-length-ascii
        if isinstance(dic[key], str):
            dic[key] = np.string_(dic[key])

        # Create dataset
        ds = h5_obj.create_dataset(key, data = dic[key])

        # Add descrption attribute
        ds.attrs['description'] = desc[key]

        # Add units attribute (if present)
        if key in units:
            ds.attrs['units'] = units[key]


#import webroot as root
from webroot import app
from flask import url_for, request, Response, render_template
from dao import article as ac
import json

@app.route('/api/article/list')
def article_get_list() :
    data = dict(ac.getList());
    if data.has_key('RECORD_0') :
        for entity in data['RECORD_0'] :
            entity['dtModifyDate'] = entity['dtModifyDate'].strftime("%Y-%m-%d %H:%M:%S")
            entity['dtCreateDate'] = entity['dtCreateDate'].strftime("%Y-%m-%d %H:%M:%S")
            #print entity.dtModifyDate.strftime("%Y-%m-%d %H:%M:%S")
            #print entity.dtCreateDate.strftime("%Y-%m-%d %H:%M:%S")

    ret_dict = {
        'result' : 'ok',
        'response' : data
    }
    header = {'content-type': 'application/json'}
    return json.dumps(ret_dict), 200, header

@app.route('/board')
def board():
    return render_template('board/list.html', data=ac.getList())


from statsmodels.tsa.stattools import adfuller, pacf_burg
from statsmodels.tsa.stattools import acf, pacf
from statsmodels.tsa.arima.model import ARIMA

import matplotlib.pyplot as plt
import numpy as np
import statistics
import pandas as pd
import warnings
from datetime import datetime
warnings.filterwarnings('ignore')

todaysDate = datetime.today().strftime('%Y-%m-%d')

fileName = 'Watchlist.csv'
dataFile = pd.read_csv(fileName, sep=',')

symbolList = list(dataFile['symbol'])

nameArray = list()
maxPACF = list()

fig = plt.figure()
#plt.figure(figsize=[15, 7.5]); # Set dimensions for figure

for i in range(0,len(symbolList)):

    tickerStr = str(symbolList[i])
    fileName = 'daily_'+tickerStr+ "_" + todaysDate
    dataFile = pd.read_csv(fileName, sep=',')

    closePrice = list(reversed(dataFile['close']))
    dates = list(reversed(dataFile['timestamp']))
    #print('length of close price array: ' + str(len(closePrice)))

    diffPrice = list()
    diffPrice.append(0)
    for n in range(1,len(closePrice)):
        diffPrice.append((closePrice[n]-closePrice[n-1]))

    # Augmented Dickey-Fuller test

    #data = pd.read_csv('jj.csv')
    #ad_fuller_result = adfuller(closePrice)
    #print(f'ADF Statistic: {ad_fuller_result[0]}')
    #print(f'p-value: {ad_fuller_result[1]}')

    acf_1 =  acf(diffPrice, nlags=7)
    pacf_1 = pacf(diffPrice, nlags=7)
    nameArray.append(tickerStr)
    maxPACF.append(max(abs(acf_1[1:])))

    plt.subplot(1,len(symbolList),(i+1))
    plt.plot(pacf_1)
    plt.xlim(1,5)
    plt.title("PACF "+ tickerStr + ", " + str(max(abs(acf_1[1:])))[0:5], size=10)

    print('The max autocorrelation coef for ' + tickerStr + 'is ' + str(max(abs(acf_1[1:]))))

plt.show()

'''
model = ARIMA(diffPrice, order=(,0,0))
model_fit = model.fit()
# print summary of fit model

model_sigma = statistics.stdev(model_fit.resid)
model_mean = statistics.mean(model_fit.resid)

fig = plt.figure()
plt.subplot(2,1,1)
plt.boxplot(model_fit.resid,bins=40)
titleStr2 = 'Price change histogram, mean: ' + str(statistics.mean(diffPrice))[0:7]
plt.title(titleStr2)
plt.show

plt.subplot(2,1,2)
plt.h(model_fit.resid,bins=40)
titleStr = 'Residuals from ARIMA, mean: ' + str(statistics.mean(model_fit.resid))[0:7]
plt.title(titleStr)
plt.show

fig = plt.figure()
plt.subplot(2,2,1)
#plt.figure(figsize=[15, 7.5]); # Set dimensions for figure
plt.plot(closePrice)
plt.title("daily SPY")
#plt.xlim([0, 1143])

plt.subplot(2,2,2)
#plt.figure(figsize=[15, 7.5]); # Set dimensions for figure
plt.plot(diffPrice)
plt.title("differenced SPY")
#plt.xlim([0, 1143])

plt.subplot(2,2,3)
plt.plot(acf_1)
plt.title("ACF plot")
plt.ylim([-0.5,0.5])

plt.subplot(2,2,4)
plt.plot(pacf_1)
plt.title("PACF plot")
plt.ylim([-0.5,0.5])
plt.show()

plt.plot(diffPrice, label="differenced price")
plt.plot(model_fit.predict(), label="model price")
plt.xlim([0,106])
plt.title("price changes: observed and predicted")
plt.show()

#some prediction stuff
castedValues = model_fit.forecast(5)
scaledXValues = range(len(diffPrice)-1,(len(diffPrice)+ len(castedValues)-1))
print(castedValues)
print(list(scaledXValues))

fig = plt.figure()
fig.subplots_adjust(hspace=.5)

plt.subplot(3,1,1)
plt.plot(closePrice, label="closing price")
plt.xlim([90,106])
plt.ylim([427,447])
plt.title("SPY Price")

plt.subplot(3,1,2)
plt.plot(diffPrice, label="differenced price")
plt.plot(model_fit.predict(), label="model price")
plt.plot(scaledXValues,castedValues, label="forecasted prices")
plt.xlim([90,106])
plt.title("price changes: observed and predicted")

plt.subplot(3,1,3)
plt.plot(scaledXValues,castedValues, label="forecasted prices")
plt.xlim([90,106])
plt.title("forecasted values")
plt.show()

'''


from database import *
from exceptions import *
from condition import Or, And, NotInCond
from itertools import product,chain,filterfalse,tee
from functools import reduce

## _____________________ Projection ____________________________________________
def project(rel, attributes):
    indices = []
    for a in attributes :
        try:
            indices.append(rel.keys[a])
        except KeyError:
            raise UnknownAttribute("key "+str(a) + " is not an attribute of relation "+ str(rel.get_keys()) )
    def projectionWithoutRepetition(iterable):
        mem = None
        for item in iterable:
            newEntry = tuple(item[i] for i in indices)
            if mem is None or mem != newEntry :
                mem = newEntry
                yield newEntry
    new = Table("projectRequest", attributes, projectionWithoutRepetition(rel.data))
    return new

## ____________________ Selection ______________________________________________
def verify_conditions(entry, cond, keys, ignoreNotIn):
    if isinstance(cond,Or):
        for clause in cond.args:
            if verify_conditions(entry,clause,keys,ignoreNotIn):
                return True
        return False

    elif isinstance(cond,And):
        for clause in cond.args:
            if not verify_conditions(entry,clause,keys,ignoreNotIn):
                return False
        return True

    elif isinstance(cond,NotInCond):
        if ignoreNotIn :
            return True
        else :
            cond = And(cond.withIn)
            return verify_conditions(entry,cond,keys,ignoreNotIn)

    else :
        return cond.eval(keys,entry)

def select(rel, condTree, ignoreNotIn=True):
    """
    Filters a table and keep only entries satisfying conditions in condTree
    """
    def is_not_previous(x,mem):
        if mem is None or x != mem :
            mem = x
            return True
        return False
    selection = lambda x : is_not_previous(x,None) and verify_conditions(x, condTree, rel.keys, ignoreNotIn)
    filtered = filter(selection , rel.data)
    new = Table("selectRequest", rel.get_keys(), filtered)
    return new

def select_distinct(rel):
    """
    Filters a table and delete duplicates
    """
    def unique_values(iterable):
        seen = set()
        for item in iterable:
            if tuple(item) not in seen:
                seen.add(tuple(item))
                yield item
    return Table("distinctProject", rel.get_keys(), unique_values(rel.data))

## ______________________ Join _________________________________________________
def join(relA, relB):
    """
    Cartesian product
    """
    def flatten(x):
        for a,b in x:
            yield a+b
    keysA = relA.get_keys()
    keysB = relB.get_keys()
    new = Table("joinRequest", keysA+keysB, flatten(product(relA.data,relB.data)))
    return new

## _______________________ Union _______________________________________________
def union(relA,relB):
    entries = chain(relA.data,relB.data)
    new = Table("unionRequest", relA.get_keys(), entries)
    return new

## ___________________________ Minus ___________________________________________
def minus(relA,relB):
    entries = filterfalse(lambda x : x in relB.get_data() , relA.data)
    new = Table("minusRequest", relA.get_keys(), entries)
    return new

## ________________________ Better Operators ___________________________________
def readSelectProjectRename(fileName, tableName, attr, isStar, conds):
    """
    Reads a CSV file, filter lines and transform remaning tuples
    keeping only the proper attributes after renaming
    """
    rel = Table.from_file(fileName)
    rel.rename(tableName)
    if conds is not None :
        rel = select(rel,conds)
    if not isStar :
        rel = project(rel,attr)
    return rel

def readSelectRenameGroupByProject(fileName, tableName, attr, isStar, conds, grpAttr, aggrOp):
    """
    Same as readSelectProjectRename but with a Group By Statement
    """
    rel = Table.from_file(fileName)
    rel.rename(tableName)
    if conds is not None :
        rel = select(rel,conds)
    rel = groupBy(rel, grpAttr, aggrOp)
    if not isStar :
        rel = project(rel,attr)
    return rel

def joinProjectRename(rel1,rel2, attr, conds):
    """
    Combines two relations in a θ-join and transform resulting tuples keeping
    only the proper attributes after renaming
    (the θ-join being a cartesian product followed by a selection)
    """
    rel = join(rel1,rel2)
    rel = select(rel,conds)
    rel = project(rel,attr)
    return rel

## _______________________ Aggregation and Group By ____________________________

def add_to_aggregate(aggr, entry, keyAggrOp):
    """
    Aggregates the entry 'entry' to entry aggr, using aggregation operators
    described in 'keyAggrOp' dict
    """
    if aggr is None :
        aggr = [None]*len(entry)
        for i in keyAggrOp.keys():
            if keyAggrOp[i]==Aggregation.COUNT:
                aggr[i]=1
            else:
                try:
                    aggr[i]=int(entry[i])
                except:
                    aggr[i]=entry[i]
    for i in keyAggrOp.keys():
        if keyAggrOp[i] == Aggregation.SUM:
            aggr[i] += int(entry[i])
        elif keyAggrOp[i] == Aggregation.MIN:
            aggr[i] = min(aggr[i], int(entry[i]))
        elif keyAggrOp[i] == Aggregation.MAX:
            aggr[i] = max(aggr[i], int(entry[i]))
        elif keyAggrOp[i] == Aggregation.COUNT:
            aggr[i] += 1
        elif keyAggrOp[i] == Aggregation.NONE:
            aggr[i]=entry[i]
    return aggr

def aggregate(rel, aggrOp):
    """
    Perform an aggregation on rel (all fields at once)
    """
    keyAggrOp = dict() # a dict int -> Aggregation operator
    for (attr,aggr) in aggrOp.items():
        keyAggrOp[rel.keys[attr]]=aggr
    data = [reduce(lambda x,y : add_to_aggregate(x,y, keyAggrOp), rel.get_data(), None)]
    rel = Table(rel.name, rel.get_keys(), iter(data))
    return rel

def groupBy(rel, grpAttr, aggrOp):
    """
    Perform a GROUP BY
    rel     : the initial table to aggregate
    grpAttr : the attribute 'GROUP BY grpAttr'
    aggrOp  : a dict  attribute->Aggregation operator
    """
    key = rel.keys[grpAttr]

    keyAggrOp = dict() # a dict int -> Aggregation operator
    for (attr,aggr) in aggrOp.items():
        keyAggrOp[rel.keys[attr]]=aggr

    datadict = dict()
    for entry in rel.get_data():
        try:
            value = int(entry[key])
        except:
            value = entry[key]
        if value not in datadict :
            datadict[value] = None
        datadict[value] = add_to_aggregate(datadict[value], entry , keyAggrOp)

    rel = Table("groupBy", rel.get_keys(), iter(datadict.values()) )
    return rel

## __________________________ Order By ____________________________

def orderBy(rel, attr, desc=False):
    """
    Order the entries of table rel according to attr.
    """

    def compKeys(x):
        l = []
        for a in attr:
            try:
                v = int(x[rel.keys[a]])
            except:
                v= x[rel.keys[a]]
            l.append(v)
        return l

    compare = lambda x : compKeys(x)
    sorted = [x for x in rel.get_data()]
    sorted.sort(key = compare, reverse=desc)
    return Table(rel.name, rel.get_keys(), iter(sorted))