import asyncio
import datetime
import json
import os
import signal
import sys
from collections import deque
from time import time
from enum import Enum

import numpy as np

import telegram_bot
import websockets

from jitted import round_, calc_diff, calc_ema, calc_cost, iter_entries, iter_long_closes, \
    iter_shrt_closes, round_dynamic

def get_keys():
    return ['inverse', 'do_long', 'do_shrt', 'qty_step', 'price_step', 'min_qty', 'min_cost',
            'contract_multiplier', 'ddown_factor', 'qty_pct', 'leverage', 'n_close_orders',
            'grid_spacing', 'pos_margin_grid_coeff', 'volatility_grid_coeff',
            'volatility_qty_coeff', 'min_markup', 'markup_range', 'ema_span', 'ema_spread',
            'stop_loss_liq_diff', 'stop_loss_pos_pct']

def sort_dict_keys(d):
    if type(d) == list:
        return [sort_dict_keys(e) for e in d]
    if type(d) != dict:
        return d
    return {key: sort_dict_keys(d[key]) for key in sorted(d)}

def flatten_dict(d, parent_key='', sep='_'):
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if type(v) == dict:
            items.extend(flatten_dict(v, new_key, sep=sep).items())
        else:
            items.append((new_key, v))
    return dict(items)

def make_get_filepath(filepath: str) -> str:
    '''
    if not is path, creates dir and subdirs for path, returns path
    '''
    dirpath = os.path.dirname(filepath) if filepath[-1] != '/' else filepath
    if not os.path.isdir(dirpath):
        os.makedirs(dirpath)
    return filepath

def load_key_secret(exchange: str, user: str) -> (str, str):
    try:
        keyfile = json.load(open('api-keys.json'))
        # Checks that the user exists, and it is for the correct exchange
        if user in keyfile and keyfile[user]["exchange"] == exchange:

            # If we need to get the `market` key:
            # market = keyfile[user]["market"]
            # print("The Market Type is " + str(market))

            keyList = [str(keyfile[user]["key"]), str(keyfile[user]["secret"])]

            return keyList
        elif user not in keyfile or keyfile[user]["exchange"] != exchange:
            print("Looks like the keys aren't configured yet, or you entered the wrong username!")
    except FileNotFoundError:
        print("File Not Found!")
        raise Exception('API KeyFile Missing!')

def print_(args, r=False, n=False):
    line = ts_to_date(time())[:19] + '  '
    str_args = '{} ' * len(args)
    line += str_args.format(*args)
    if n:
        print('\n' + line, end=' ')
    elif r:
        print('\r' + line, end=' ')
    else:
        print(line)
    return line

def ts_to_date(timestamp: float) -> str:
    return str(datetime.datetime.fromtimestamp(timestamp)).replace(' ', 'T')

def filter_orders(actual_orders: [dict],
                  ideal_orders: [dict],
                  keys: [str] = ['symbol', 'side', 'qty', 'price']) -> ([dict], [dict]):
    # returns (orders_to_delete, orders_to_create)

    if not actual_orders:
        return [], ideal_orders
    if not ideal_orders:
        return actual_orders, []
    actual_orders = actual_orders.copy()
    orders_to_create = []
    ideal_orders_cropped = [{k: o[k] for k in keys} for o in ideal_orders]
    actual_orders_cropped = [{k: o[k] for k in keys} for o in actual_orders]
    for ioc, io in zip(ideal_orders_cropped, ideal_orders):
        matches = [(aoc, ao) for aoc, ao in zip(actual_orders_cropped, actual_orders) if aoc == ioc]
        if matches:
            actual_orders.remove(matches[0][1])
            actual_orders_cropped.remove(matches[0][0])
        else:
            orders_to_create.append(io)
    return actual_orders, orders_to_create

def flatten(lst: list) -> list:
    return [y for x in lst for y in x]

class Bot:
    def __init__(self, user: str, config: dict, telegram = None):
        self.config = config
        self.user = user
        self.telegram = telegram

        for key in config:
            setattr(self, key, config[key])

        self.ema_span = int(round(self.ema_span))
        self.ema_alpha = 2 / (self.ema_span + 1)
        self.ema_alpha_ = 1 - self.ema_alpha

        self.ts_locked = {'cancel_orders': 0, 'decide': 0, 'update_open_orders': 0,
                          'update_position': 0, 'print': 0, 'create_orders': 0}
        self.ts_released = {k: 1 for k in self.ts_locked}

        self.position = {}
        self.open_orders = []
        self.highest_bid = 0.0
        self.lowest_ask = 9.9e9
        self.price = 0
        self.ob = [0.0, 0.0]
        self.ema = 0.0

        self.n_open_orders_limit = 8
        self.last_price_diff_limit = 0.15
        self.n_orders_per_execution = 4

        self.hedge_mode = True
        self.contract_multiplier = self.config['contract_multiplier'] = 1.0

        self.log_filepath = make_get_filepath(f"logs/{self.exchange}/{config['config_name']}.log")

        self.key, self.secret = load_key_secret(config['exchange'], user)

        self.log_level = 0

        self.stop_websocket = False
        self.process_websocket_ticks = True

    def set_config(self, config):
        config['ema_span'] = int(round(config['ema_span']))
        self.config = config
        for key in config:
            setattr(self, key, config[key])
            if key in self.xk:
                self.xk[key] = config[key]

    def set_config_value(self, key, value):
        self.config[key] = value
        setattr(self, key, self.config[key])

    async def _init(self):
        self.xk = {k: float(self.config[k]) for k in get_keys()}

    def dump_log(self, data) -> None:
        if self.config['logging_level'] > 0:
            with open(self.log_filepath, 'a') as f:
                f.write(json.dumps({**{'log_timestamp': time()}, **data}) + '\n')

    async def update_open_orders(self) -> None:
        if self.ts_locked['update_open_orders'] > self.ts_released['update_open_orders']:
            return
        try:
            open_orders = await self.fetch_open_orders()
            self.highest_bid, self.lowest_ask = 0.0, 9.9e9
            for o in open_orders:
                if o['side'] == 'buy':
                    self.highest_bid = max(self.highest_bid, o['price'])
                elif o['side'] == 'sell':
                    self.lowest_ask = min(self.lowest_ask, o['price'])
            if self.open_orders != open_orders:
                self.dump_log({'log_type': 'open_orders', 'data': open_orders})
            self.open_orders = open_orders
            self.ts_released['update_open_orders'] = time()
        except Exception as e:
            print('error with update open orders', e)

    async def update_position(self) -> None:
        # also updates open orders
        if self.ts_locked['update_position'] > self.ts_released['update_position']:
            return
        self.ts_locked['update_position'] = time()
        try:
            position, _ = await asyncio.gather(self.fetch_position(),
                                               self.update_open_orders())
            position['used_margin'] = \
                ((calc_cost(position['long']['size'], position['long']['price'],
                            self.xk['inverse'], self.xk['contract_multiplier'])
                  if position['long']['price'] else 0.0) +
                 (calc_cost(position['shrt']['size'], position['shrt']['price'],
                            self.xk['inverse'], self.xk['contract_multiplier'])
                  if position['shrt']['price'] else 0.0)) / self.leverage
            position['available_margin'] = (position['equity'] - position['used_margin']) * 0.9
            position['long']['liq_diff'] = calc_diff(position['long']['liquidation_price'], self.price)
            position['shrt']['liq_diff'] = calc_diff(position['shrt']['liquidation_price'], self.price)
            if self.position != position:
                self.dump_log({'log_type': 'position', 'data': position})
            self.position = position
            self.ts_released['update_position'] = time()
        except Exception as e:
            print('error with update position', e)

    async def create_orders(self, orders_to_create: [dict]) -> dict:
        if not orders_to_create:
            return
        if self.ts_locked['create_orders'] > self.ts_released['create_orders']:
            return
        self.ts_locked['create_orders'] = time()
        creations = []
        for oc in sorted(orders_to_create, key=lambda x: x['qty']):
            try:
                creations.append((oc, asyncio.create_task(self.execute_order(oc))))
            except Exception as e:
                print_(['error creating order a', oc, e], n=True)
        created_orders = []
        for oc, c in creations:
            try:
                o = await c
                created_orders.append(o)
                if 'side' in o:
                    print_([' created order', o['symbol'], o['side'], o['position_side'], o['qty'],
                            o['price']], n=True)
                else:
                    print_(['error creating order b', o, oc], n=True)
                self.dump_log({'log_type': 'create_order', 'data': o})
            except Exception as e:
                print_(['error creating order c', oc, c.exception(), e], n=True)
                self.dump_log({'log_type': 'create_order', 'data': {'result': str(c.exception()),
                                                                    'error': repr(e), 'data': oc}})
        self.ts_released['create_orders'] = time()
        return created_orders

    async def cancel_orders(self, orders_to_cancel: [dict]) -> [dict]:
        if not orders_to_cancel:
            return
        if self.ts_locked['cancel_orders'] > self.ts_released['cancel_orders']:
            return
        self.ts_locked['cancel_orders'] = time()
        deletions = []
        for oc in orders_to_cancel:
            try:
                deletions.append((oc,
                                  asyncio.create_task(self.execute_cancellation(oc))))
            except Exception as e:
                print_(['error cancelling order a', oc, e])
        canceled_orders = []
        for oc, c in deletions:
            try:
                o = await c
                canceled_orders.append(o)
                if 'side' in o:
                    print_(['cancelled order', o['symbol'], o['side'], o['position_side'], o['qty'],
                            o['price']], n=True)
                else:
                    print_(['error cancelling order', o], n=True)
                self.dump_log({'log_type': 'cancel_order', 'data': o})
            except Exception as e:
                print_(['error cancelling order b', oc, c.exception(), e], n=True)
                self.dump_log({'log_type': 'cancel_order', 'data': {'result': str(c.exception()),
                                                                    'error': repr(e), 'data': oc}})
        self.ts_released['cancel_orders'] = time()
        return canceled_orders

    def stop(self) -> None:
        self.stop_websocket = True

    def pause(self) -> None:
        self.process_websocket_ticks = False

    def resume(self) -> None:
        self.process_websocket_ticks = True

    def calc_orders(self):
        balance = self.position['wallet_balance'] * 0.9
        long_psize = self.position['long']['size']
        long_pprice = self.position['long']['price']
        shrt_psize = self.position['shrt']['size']
        shrt_pprice = self.position['shrt']['price']

        if self.hedge_mode:
            do_long = self.do_long or long_psize != 0.0
            do_shrt = self.do_shrt or shrt_psize != 0.0
        else:
            no_pos = long_psize == 0.0 and shrt_psize == 0.0
            do_long = (no_pos and self.do_long) or long_psize != 0.0
            do_shrt = (no_pos and self.do_shrt) or shrt_psize != 0.0

        self.xk['do_long'] = do_long
        self.xk['do_shrt'] = do_shrt

        liq_price = self.position['long']['liquidation_price'] if long_psize > abs(shrt_psize) \
            else self.position['shrt']['liquidation_price']

        long_entry_orders, shrt_entry_orders, long_close_orders, shrt_close_orders = [], [], [], []
        stop_loss_close = False

        for tpl in iter_entries(balance, long_psize, long_pprice, shrt_psize, shrt_pprice,
                                liq_price, self.ob[0], self.ob[1], self.ema, self.price,
                                self.volatility, **self.xk):
            if (len(long_entry_orders) >= self.n_open_orders_limit and
                len(shrt_entry_orders) >= self.n_open_orders_limit) or \
                    calc_diff(tpl[1], self.price) > self.last_price_diff_limit:
                break
            if tpl[4] == 'stop_loss_shrt_close':
                shrt_close_orders.append({'side': 'buy', 'position_side': 'shrt', 'qty': abs(tpl[0]),
                                          'price': tpl[1], 'type': 'limit', 'reduce_only': True,
                                          'custom_id': tpl[4]})
                shrt_psize = tpl[2]
                stop_loss_close = True
            elif tpl[4] == 'stop_loss_long_close':
                long_close_orders.append({'side': 'sell', 'position_side': 'long', 'qty': abs(tpl[0]),
                                          'price': tpl[1], 'type': 'limit', 'reduce_only': True,
                                          'custom_id': tpl[4]})
                long_psize = tpl[2]
                stop_loss_close = True
            elif tpl[0] > 0.0:
                long_entry_orders.append({'side': 'buy', 'position_side': 'long', 'qty': tpl[0],
                                          'price': tpl[1], 'type': 'limit', 'reduce_only': False,
                                          'custom_id': tpl[4]})
            else:
                shrt_entry_orders.append({'side': 'sell', 'position_side': 'shrt', 'qty': abs(tpl[0]),
                                          'price': tpl[1], 'type': 'limit', 'reduce_only': False,
                                          'custom_id': tpl[4]})

        for ask_qty, ask_price, _ in iter_long_closes(balance, long_psize, long_pprice, self.ob[1],
                                                      **self.xk):
            if len(long_close_orders) >= self.n_open_orders_limit or \
                    calc_diff(ask_price, self.price) > self.last_price_diff_limit or \
                    stop_loss_close:
                break
            long_close_orders.append({'side': 'sell', 'position_side': 'long', 'qty': abs(ask_qty),
                                      'price': float(ask_price), 'type': 'limit',
                                      'reduce_only': True, 'custom_id': 'close'})

        for bid_qty, bid_price, _ in iter_shrt_closes(balance, shrt_psize, shrt_pprice, self.ob[0],
                                                      **self.xk):
            if len(shrt_close_orders) >= self.n_open_orders_limit or \
                    calc_diff(bid_price, self.price) > self.last_price_diff_limit or \
                    stop_loss_close:
                break
            shrt_close_orders.append({'side': 'buy', 'position_side': 'shrt', 'qty': abs(bid_qty),
                                      'price': float(bid_price), 'type': 'limit',
                                      'reduce_only': True, 'custom_id': 'close'})
        return long_entry_orders + shrt_entry_orders + long_close_orders + shrt_close_orders

    async def cancel_and_create(self):
        await asyncio.sleep(0.005)
        await self.update_position()
        await asyncio.sleep(0.005)
        if any([self.ts_locked[k_] > self.ts_released[k_]
                for k_ in [x for x in self.ts_locked if x != 'decide']]):
            return
        to_cancel, to_create = filter_orders(self.open_orders,
                                             self.calc_orders(),
                                             keys=['side', 'position_side', 'qty', 'price'])
        to_cancel = sorted(to_cancel, key=lambda x: calc_diff(x['price'], self.price))
        to_create = sorted(to_create, key=lambda x: calc_diff(x['price'], self.price))
        results = []
        if to_cancel:
            results.append(asyncio.create_task(self.cancel_orders(to_cancel[:self.n_orders_per_execution])))
            await asyncio.sleep(0.005)  # sleep 5 ms between sending cancellations and creations
        if to_create:
            results.append(await self.create_orders(to_create[:self.n_orders_per_execution]))
        await asyncio.sleep(0.005)
        await self.update_position()
        if any(results):
            print()
        return results

    async def decide(self):
        if self.price <= self.highest_bid:
            self.ts_locked['decide'] = time()
            print_(['bid maybe taken'], n=True)
            await self.cancel_and_create()
            self.ts_released['decide'] = time()
            return
        if self.price >= self.lowest_ask:
            self.ts_locked['decide'] = time()
            print_(['ask maybe taken'], n=True)
            await self.cancel_and_create()
            self.ts_released['decide'] = time()
            return
        if time() - self.ts_locked['decide'] > 5:
            self.ts_locked['decide'] = time()
            await self.cancel_and_create()
            self.ts_released['decide'] = time()
            return
        if time() - self.ts_released['print'] >= 0.5:
            self.ts_released['print'] = time()
            line = f"{self.symbol} "
            line += f"long {self.position['long']['size']} @ "
            line += f"{round_(self.position['long']['price'], self.price_step)} "
            long_closes = sorted([o for o in self.open_orders if o['side'] == 'sell'
                                  and o['position_side'] == 'long'], key=lambda x: x['price'])
            long_entries = sorted([o for o in self.open_orders if o['side'] == 'buy'
                                   and o['position_side'] == 'long'], key=lambda x: x['price'])
            line += f"close @ {long_closes[0]['price'] if long_closes else 0.0} "
            line += f"enter @ {long_entries[-1]['price'] if long_entries else 0.0} "
            line += f"|| shrt {self.position['shrt']['size']} @ "
            line += f"{round_(self.position['shrt']['price'], self.price_step)} "
            shrt_closes = sorted([o for o in self.open_orders if o['side'] == 'buy'
                                  and (o['position_side'] == 'shrt' or
                                       (o['position_side'] == 'both' and
                                        self.position['shrt']['size'] != 0.0))],
                                 key=lambda x: x['price'])
            shrt_entries = sorted([o for o in self.open_orders if o['side'] == 'sell'
                                   and (o['position_side'] == 'shrt' or
                                        (o['position_side'] == 'both' and
                                         self.position['shrt']['size'] != 0.0))],
                                  key=lambda x: x['price'])
            line += f"close @ {shrt_closes[-1]['price'] if shrt_closes else 0.0} "
            line += f"enter @ {shrt_entries[0]['price'] if shrt_entries else 0.0} "
            if self.position['long']['size'] > abs(self.position['shrt']['size']):
                liq_price = self.position['long']['liquidation_price']
                sl_trigger_price = round_(liq_price / (1 - self.stop_loss_liq_diff), self.price_step)
            else:
                liq_price = self.position['shrt']['liquidation_price']
                sl_trigger_price = round_(liq_price / (1 + self.stop_loss_liq_diff), self.price_step)
            line += f"|| last {self.price} liq {round_dynamic(liq_price, 3)} "
            line += f"sl trig {round_dynamic(sl_trigger_price, 3)} "
            line += f"ema {round_dynamic(self.ema, 3)} "
            line += f"bal {round_dynamic(self.position['wallet_balance'], 3)} "
            line += f"equity {round_dynamic(self.position['equity'], 3)} "
            line += f"v. {round_dynamic(self.volatility, 3)} "
            print_([line], r=True)

    def flush_stuck_locks(self, timeout: float = 4.0) -> None:
        now = time()
        for key in self.ts_locked:
            if self.ts_locked[key] > self.ts_released[key]:
                if now - self.ts_locked[key] > timeout:
                    print('flushing', key)
                    self.ts_released[key] = now

    async def fetch_compressed_ticks(self):

        def drop_consecutive_same_prices(ticks_):
            compressed_ = [ticks_[0]]
            for i in range(1, len(ticks_)):
                if ticks_[i]['price'] != compressed_[-1]['price']:
                    compressed_.append(ticks_[i])
            return compressed_

        ticks_unabridged = await self.fetch_ticks(do_print=False)
        ticks_per_fetch = len(ticks_unabridged)
        ticks = drop_consecutive_same_prices(ticks_unabridged)
        if self.exchange == 'bybit' and self.market_type == 'linear_perpetual':
            print('\nwarning:  bybit linear usdt symbols only allows fetching most recent 1000 ticks')
            return ticks
        delay_between_fetches = 0.55
        print()
        while True:
            print(f'\rfetching ticks... {len(ticks)} of {self.ema_span} ', end= ' ')
            sts = time()
            new_ticks = await self.fetch_ticks(from_id=ticks[0]['trade_id'] - ticks_per_fetch,
                                               do_print=False)
            wait_for = max(0.0, delay_between_fetches - (time() - sts))
            ticks = drop_consecutive_same_prices(sorted(new_ticks + ticks,
                                                        key=lambda x: x['trade_id']))
            if len(ticks) > self.ema_span:
                break
            await asyncio.sleep(wait_for)
        new_ticks = await self.fetch_ticks(do_print=False)
        return drop_consecutive_same_prices(sorted(ticks + new_ticks, key=lambda x: x['trade_id']))

    async def init_indicators(self):
        ticks = await self.fetch_compressed_ticks()
        ema = ticks[0]['price']
        self.tick_prices_deque = deque(maxlen=self.ema_span)
        for tick in ticks:
            self.tick_prices_deque.append(tick['price'])
            ema = ema * self.ema_alpha_ + tick['price'] * self.ema_alpha
        if len(self.tick_prices_deque) < self.ema_span:
            print('\nwarning: insufficient ticks fetched, filling deque with duplicate ticks...')
            print('ema and volatility will be inaccurate until deque is filled with websocket ticks')
            while len(self.tick_prices_deque) < self.ema_span:
                self.tick_prices_deque.extend([t['price'] for t in ticks])
        self.ema = ema
        self.sum_prices = sum(self.tick_prices_deque)
        self.sum_prices_squared = sum([e ** 2 for e in self.tick_prices_deque])
        self.price_std = np.sqrt((self.sum_prices_squared / len(self.tick_prices_deque) -
                                 ((self.sum_prices / len(self.tick_prices_deque)) ** 2)))
        self.volatility = self.price_std / self.ema
        print('\ndebug len ticks, prices deque, ema_span')
        print(len(ticks), len(self.tick_prices_deque), self.ema_span)

    def update_indicators(self, ticks: dict):
        for tick in ticks:
            if tick['is_buyer_maker']:
                self.ob[0] = tick['price']
            else:
                self.ob[1] = tick['price']
            self.ema = calc_ema(self.ema_alpha, self.ema_alpha_, self.ema, tick['price'])
            self.sum_prices -= self.tick_prices_deque[0]
            self.sum_prices_squared -= self.tick_prices_deque[0] ** 2
            self.tick_prices_deque.append(tick['price'])
            self.sum_prices += self.tick_prices_deque[-1]
            self.sum_prices_squared += self.tick_prices_deque[-1] ** 2
        self.price_std = np.sqrt((self.sum_prices_squared / len(self.tick_prices_deque) -
                                 ((self.sum_prices / len(self.tick_prices_deque)) ** 2)))
        self.price = ticks[-1]['price']
        self.volatility = self.price_std / self.ema

    async def start_websocket(self) -> None:
        self.stop_websocket = False
        self.process_websocket_ticks = True
        print_([self.endpoints['websocket']])
        await self.update_position()
        await self.init_exchange_config()
        await self.init_indicators()
        k = 1
        async with websockets.connect(self.endpoints['websocket']) as ws:
            await self.subscribe_ws(ws)
            async for msg in ws:
                if msg is None:
                    continue
                try:
                    ticks = self.standardize_websocket_ticks(json.loads(msg))
                    if self.process_websocket_ticks:
                        if ticks:
                            self.update_indicators(ticks)
                        if self.ts_locked['decide'] < self.ts_released['decide']:
                            asyncio.create_task(self.decide())
                    if k % 10 == 0:
                        self.flush_stuck_locks()
                        k = 1
                    if self.stop_websocket:
                        if self.telegram is not None:
                            self.telegram.send_msg("<pre>Bot stopped</pre>")
                        break
                    k += 1

                except Exception as e:
                    if 'success' not in msg:
                        print('error in websocket', e, msg)

async def start_bot(bot):
    await bot.start_websocket()

async def create_binance_bot(user: str, config: str):
    from binance import BinanceBot
    bot = BinanceBot(user, config)
    await bot._init()
    return bot

async def create_bybit_bot(user: str, config: str):
    from bybit import Bybit
    bot = Bybit(user, config)
    await bot._init()
    return bot

async def _start_telegram(account: dict, bot: Bot):
    telegram = telegram_bot.Telegram(token=account['telegram']['token'],
                                     chat_id=account['telegram']['chat_id'],
                                     bot=bot,
                                     loop=asyncio.get_event_loop())
    telegram.log_start()
    return telegram

async def main() -> None:
    try:
        accounts = json.load(open('api-keys.json'))
    except Exception as e:
        print(e, 'failed to load api-keys.json file')
        return
    if sys.argv[1] in accounts:
        account = accounts[sys.argv[1]]
    else:
        print('unrecognized account name', sys.argv[1])
        return
    try:
        config = json.load(open(sys.argv[3]))
    except Exception as e:
        print(e, 'failed to load config', sys.argv[3])
        return
    config['exchange'] = account['exchange']
    config['symbol'] = sys.argv[2]

    if account['exchange'] == 'binance':
        bot = await create_binance_bot(sys.argv[1], config)
    elif account['exchange'] == 'bybit':
        bot = await create_bybit_bot(sys.argv[1], config)
    else:
        raise Exception('unknown exchange', account['exchange'])
    print('using config')
    print(json.dumps(config, indent=4))

    if 'telegram' in account and account['telegram']['enabled']:
        telegram = await _start_telegram(account=account, bot=bot)
        signal.signal(signal.SIGINT, telegram.exit)
    await start_bot(bot)

if __name__ == '__main__':
    asyncio.run(main())


import os
from PIL import ImageOps
from PIL import Image
from argparse import ArgumentParser
from tqdm import tqdm
import numpy as np
import os
import face_recognition
import torch
import torchvision.transforms as transforms

ACCEPTED_IMAGE_EXTENSIONS = ['.jpg']

parser = ArgumentParser(description="Getting face landmarks out of images of faces")

parser.add_argument("--data-dir", required=True, type=str,
                    help="Absolute path to the directory where image files are located")

parser.add_argument("--save-dir", required=True, type=str,
                    help="Absolute path to the directory where face landmarks will be saved")

def load_image(file_path):
    image = Image.open(str(file_path))
    image = ImageOps.exif_transpose(image)
    return image

def save_face_landmarks(args, root, file_name, face_landmarks):
    additional_dirs = os.path.relpath(root, start=args.data_dir)
    save_dir = os.path.join(args.save_dir, additional_dirs)
    os.makedirs(save_dir,  exist_ok=True)
    name, _ = os.path.splitext(file_name)
    name += "_landmarks.npy"
    new_file_path = os.path.join(save_dir, name)
    np.save(new_file_path, face_landmarks)

def calculate_landmarks(image):
    landmarks = face_recognition.face_landmarks(image)
    landmarks_tensor = torch.empty(0)
    if not landmarks:
        return None
    for key in landmarks[0].keys():
        landmarks_tensor = torch.cat((landmarks_tensor, torch.tensor(landmarks[0][key])))
    landmarks_tensor = landmarks_tensor.ravel()
    return landmarks_tensor

def main():
    args = parser.parse_args()

    transform = transforms.Compose([
        transforms.Resize((224, 224)),
        transforms.ToTensor()
    ])

    for root, _, files in tqdm(os.walk(args.data_dir), desc="Outer Loop"):
        for file_name in tqdm(files, desc="Inner Loop", leave=False):
            _, extension = os.path.splitext(file_name)
            if extension not in ACCEPTED_IMAGE_EXTENSIONS:
                continue
            file_path = os.path.join(root, file_name)
            image = load_image(file_path)
            img_tensor = transform(image).permute(1, 2, 0)
            img_tensor = img_tensor * 255
            img_tensor = img_tensor.to(torch.int)
            img = np.array(img_tensor, dtype=np.uint8)
            face_landmarks = calculate_landmarks(img)
            if face_landmarks is not None:
                save_face_landmarks(args, root, file_name, face_landmarks)

if __name__ == "__main__":
    main()


"""
Module for manipulating rasters, including adding padding to and splitting rasters.
"""
import rasterio as rio
from itertools import product
from rasterio import band, windows

from rasterio.windows import Window

import numpy as np

import math
import os

import rasterio

from osgeo import gdal

def add_padding_to_raster(raster_in: str, raster_out: str, dimension: int, verbose = True):
    """Adds a padding to the raster, based on the dimension of the tiles.

    Args:
        raster_in (str): Filename of input raster.
        raster_out (str): Filename of output raster.
        dimension (int): Dimension of tiles.

    Example:
        The raster is 118 x 135 and has a spatial resolution of 10 metre.
        The dimension of the tiles is set at 1 kilometre, which is 100 pixels (1000 / 10).
        The raster covers an area of 1180 x 1350 metre.
        The padding should therefore be (x,y): 10, 25.
        This would give a raster of 1180 + 2 * 10 = 1200 and 1350 + 2 * 25 = 1400.
    """
    assert type(dimension) == int, "The dimension has to be provided as integer."
    assert os.path.exists(raster_in), "The file '{}' does not exist.".format(raster_in)

    # Open raster
    raster = rio.open(raster_in)

    # Get spatial resolution of raster
    raster_spatial_resolution = get_spatial_resolution_raster(raster_in)

    # Calculate resolution of tile in pixels
    tile_resolution = dimension / raster_spatial_resolution

    # Create list for bands
    padded_bands = []

    # Copy metadata
    out_meta = raster.meta.copy()

    # Determine padding based on spatial resolution of raster
    new_raster_width = raster.width + (tile_resolution - raster.width % tile_resolution)
    new_raster_height = raster.height + (tile_resolution - raster.height % tile_resolution)

    if verbose:
        print("The raster width will be increased from {} to {} pixels.".format(raster.width, new_raster_width))
        print("The raster height will be increased from {} to {} pixels.\n".format(raster.height, new_raster_height))

    # Determine padding
    padding_x = int(new_raster_width - raster.width)
    padding_y = int(new_raster_height - raster.height)

    # Update values based on padding
    out_meta.update({
        'width': new_raster_width,
        'height': new_raster_height
    })

    # Loop through raster bands
    for band_no in range(1, raster.count + 1):
        print("Processing band no. {}...".format(band_no))

        # Read band
        raster_band = raster.read(band_no)

        # Add padding
        padded_raster_band = np.pad(
            raster_band,
            pad_width = ((0, padding_y), (0, padding_x)),
            mode = 'constant',
            constant_values = 0)

        # Append padded raster band to list
        padded_bands.append(padded_raster_band)

    with rio.open(raster_out, 'w', **out_meta) as dest:
        for band_nr, src in enumerate(padded_bands, start=1):
            dest.write(src, band_nr)

def get_spatial_resolution_raster(raster_fpath, verbose = False) -> float:
    """Extracts spatial resolution from raster.

    Args:
        raster_fpath (str): Filepath of raster file.
        verbose (bool): Verbose output.

    Returns:
        float: Spatial resolution.
    """
    if os.path.exists(raster_fpath):
        assert os.path.splitext(raster_fpath)[1] == '.tif', "The file should be .tif format."

        # Open raster
        with rio.open(raster_fpath) as inds:
            spatial_resolution = inds.profile['transform'][0]

            if verbose:
                print("The spatial resolution of '{}' is {} {}(s).".format(
                    raster_fpath,
                    spatial_resolution,
                    inds.crs.linear_units))

            return spatial_resolution
    else:
        raise FileNotFoundError(
            "The raster file does not exist."
        )

def get_tiles(ds, width = 256, height = 256):
    nols, nrows = ds.meta['width'], ds.meta['height']
    offsets = product(range(0, nols, width), range(0, nrows, height))
    big_window = windows.Window(col_off=0, row_off=0, width=nols, height=nrows)
    for col_off, row_off in  offsets:
        window = windows.Window(col_off=col_off, row_off=row_off, width=width, height=height).intersection(big_window)
        transform = windows.transform(window, ds.transform)
        yield window, transform

def tile_raster(raster_in, raster_out, dimension: int = 5000, verbose = True):
    """Splits raster in smaller tiles.

    Args:
        raster ([type]): [description]
        resolution ([type]): [description]
    """
    # Check if raster exists
    if not os.path.exists(raster_in):
        raise FileNotFoundError(
            "The file '{}' could not be found.".format(raster_in)
        )

    # CHeck if output folder exists
    if not os.path.exists(raster_out):
        if verbose:
            print("Creating new folder for tiles...")
        os.mkdir("data/intermediate/tiles")

    # Open raster
    with rio.open(raster_in) as raster:
        if verbose:
            print('The CRS of {} is {}.'.format(raster_in, raster.crs))

        # Get spatial resolution
        spatial_resolution = get_spatial_resolution_raster(raster_in)

        # Calculate the tile size in pixels
        tile_pixel_size = int(dimension / spatial_resolution)

        # Calculate number of tiles in width and length
        num_tiles_x = math.ceil(raster.width / tile_pixel_size)
        num_tiles_y = math.ceil(raster.height / tile_pixel_size)

        if verbose:
            print("There will be created {} tiles in total (width: {}, height: {}".format(
                num_tiles_x * num_tiles_y,
                num_tiles_x,
                num_tiles_y
            ))

        meta = raster.meta.copy()

        for window, transform in get_tiles(raster, width = tile_pixel_size, height = tile_pixel_size):
            meta['transform'] = transform
            meta['width'], meta['height'] = window.width, window.height
            output_filename = 'tile_{}-{}.tif'
            outpath = os.path.join(raster_out, output_filename.format(int(window.col_off), int(window.row_off)))
            with rio.open(outpath, 'w', **meta) as outds:
                outds.write(raster.read(window = window))

def delete_single_value_tiles(tiles_folder: str) -> None:
    """Deletes raster files with only a single-value.

    Args:
        tiles_folder (str): Folder of rasters.
    """
    # Loop through tiles
    for tile in os.listdir(tiles_folder):
        # Combine paths of input folder and raster filename
        f = os.path.join(tiles_folder, tile)

        # If tile exists ...
        if os.path.isfile(f) and os.path.splitext(f)[1] == ".tif":

            # Open raster and read band
            raster = rio.open(f)
            raster_band = raster.read(1)

            # If all values are the same, remove file
            if (np.all(raster_band == raster_band[0])):
                os.remove(f)

def set_band_descriptions(raster_fpath: str, bands_txt: str) -> None:
    """Sets descriptions of raster bands using a .txt file.

    Args:
        raster_fpath (str): Filepath of raster.
        bands_txt (str): Filepath of .txt file with band names.
    """
    # Open raster using GDAL
    ds = gdal.Open(raster_fpath, gdal.GA_Update)

    # Open .txt file
    band_file = open(bands_txt, "r")

    # Splitting the content by comma
    bands = band_file.read().split(sep = ",")

    # Loop through all bands
    for band in range(1, len(bands)+1):
        rb = ds.GetRasterBand(band)
        rb.SetDescription(bands[band - 1])

    # Delete dataset
    del ds

def combine_rasters(rasters: list):
    print("There are {} rasters to be combined.".format(len(rasters)))

    for raster_fpath in rasters:
        raster = rio.open(raster_fpath)

        for band in range(1, raster.count):
            print(band)


#! /usr/bin/env python
# -*- coding: utf-8 -*-

from django.shortcuts import render, HttpResponseRedirect, HttpResponse
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required
from appconf.models import Project
from appconf.forms import ProjectForm
from accounts.permission import permission_verify
import csv
import datetime
from cmdb.api import str2gb
from delivery.models import Delivery

@login_required()
@permission_verify()
def project_list(request):
    all_project = Project.objects.all()
    results = {
        'all_project':  all_project,
    }
    return render(request, 'appconf/project_list.html', results)

@login_required
@permission_verify()
def project_del(request):
    project_id = request.GET.get('id', '')
    if project_id:
        Project.objects.filter(id=project_id).delete()

    if request.method == 'POST':
        project_items = request.POST.getlist('g_check', [])
        if project_items:
            for n in project_items:
                Project.objects.filter(id=n).delete()
    # all_product = Project.objects.all()
    return HttpResponseRedirect(reverse('project_list'))

@login_required
@permission_verify()
def project_add(request):
    if request.method == 'POST':
        form = ProjectForm(request.POST)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(reverse('project_list'))
    else:
        form = ProjectForm()

    results = {
        'form': form,
        'request': request,
    }
    return render(request, 'appconf/project_base.html', results)

@login_required
@permission_verify()
def project_edit(request, project_id):
    project = Project.objects.get(id=project_id)
    if request.method == 'POST':
        form = ProjectForm(request.POST, instance=project)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect(reverse('project_list'))
    else:
        form = ProjectForm(instance=project)

    results = {
        'form': form,
        'project_id': project_id,
        'request': request,
    }
    return render(request, 'appconf/project_base.html', results)

@login_required
@permission_verify()
def project_export(request):
    export = request.GET.get("export", '')
    project_id_list = request.GET.getlist("id", '')
    if export == "part":
        if project_id_list:
            project_find = []
            for project_id in project_id_list:
                project_item = Project.objects.get(id=project_id)
                if project_item:
                    project_find.append(project_item)

    if export == "all":
        project_find = Project.objects.all()

    response = HttpResponse(content_type='text/csv')
    now = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M')
    file_name = 'adminset_project_' + now + '.csv'
    response['Content-Disposition'] = "attachment; filename="+file_name
    writer = csv.writer(response)
    writer.writerow([str2gb(u'项目名称'), str2gb(u'项目描述'), str2gb(u'语言类型'), str2gb(u'程序类型'),
                     str2gb(u'服务器类型'), str2gb(u'程序框架'), str2gb(u'源类型'), str2gb(u'源地址'),
                     str2gb(u'程序部署路径'), str2gb(u'配置文件路径'),
                     str2gb(u'所属产品线'), str2gb(u'项目负责人'), str2gb(u'服务器')])
    for p in project_find:
        server_result = ""
        try:
            p2 = Delivery.objects.get(job_name_id=p.id)
            for server in p2.serverList.all():
                server_result += server.hostname+"\n"
        except:
            server_result = ""
        writer.writerow([str2gb(p.name), str2gb(p.description), p.language_type, p.app_type, p.server_type,
                        p.app_arch, p.source_type, p.source_address, p.appPath, p.configPath, str2gb(p.product),
                         str2gb(p.owner), str2gb(server_result)])
    return response


import matplotlib.pyplot as plt
import pandas as pd

# fig = plt.figure()  # 定义画图区域
# ax1 = fig.add_subplot(2, 2, 1)
# ax2 = fig.add_subplot(2, 2, 2)
# ax3 = fig.add_subplot(2, 2, 4)
# plt.show()

import numpy as np

# figsize控制子图大小
fig = plt.figure(figsize=(3, 3))
ax1 = fig.add_subplot(2, 2, 1)
ax2 = fig.add_subplot(2, 2, 2)

ax1.plot(np.random.randint(1, 5, 5), np.arange(5))
plt.show()

unrate = pd.read_csv('unrate.csv')
fig = plt.figure(figsize=(10, 6))
colors = ['red', 'blue', 'green', 'orange', 'black']
for i in range(5):
    start_index = i * 12
    end_index = (i + 1) * 12
    subset = unrate[start_index:end_index]
    label = str(1948 + i)
    plt.plot(subset['MONTH'], subset['VALUE'], c=colors[i], label=label)
plt.legend(loc='best')
plt.show()


import torch.optim.lr_scheduler
from bootstrap.lib.logger import Logger

class LearningRateScheduler():

    def __init__(self, optimizer, engine=None):
        self.optimizer = optimizer
        self.scheduler = None
        self.param_groups = self.optimizer.param_groups  # to keep compatibility with GradClipper
        if engine:
            engine.register_hook('train_on_start_epoch', self.step_scheduler)

    def step(self, closure=None):
        self.optimizer.step(closure)

    def zero_grad(self):
        self.optimizer.zero_grad()

    def state_dict(self):
        state = {}
        state['optimizer'] = self.optimizer.state_dict()
        state['scheduler'] = self.scheduler.state_dict()
        return state

    def load_state_dict(self, state):
        self.optimizer.load_state_dict(state['optimizer'])
        self.scheduler.load_state_dict(state['scheduler'])

    def step_scheduler(self):
        self.scheduler.step()
        Logger().log_value('train_epoch.lr', self.optimizer.param_groups[0]['lr'])

class StepLR(LearningRateScheduler):

    def __init__(self, optimizer, engine=None, step_size=None, gamma=0.1, last_epoch=-1):
        super(StepLR, self).__init__(optimizer, engine)
        self.scheduler = torch.optim.lr_scheduler.StepLR(
            optimizer,
            step_size,
            gamma=gamma,
            last_epoch=last_epoch)

class CosineAnnealingLR(LearningRateScheduler):

    def __init__(self, optimizer, engine=None, T_max=None, eta_min=0, last_epoch=-1):
        super(CosineAnnealingLR, self).__init__(optimizer, engine)
        self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
            optimizer,
            T_max,
            eta_min=eta_min,
            last_epoch=last_epoch)


#importing library
from collections import Counter

#computing frequency of each note
freq = dict(Counter(notes_))

#library for visualiation
import matplotlib.pyplot as plt

#consider only the frequencies
no=[count for _,count in freq.items()]

#set the figure size
plt.figure(figsize=(5,5))

#plot
plt.hist(no)

#From the above plot, we can infer that most of the notes have a very low frequency.
#So, let us keep the top frequent notes and ignore the low-frequency ones.
#Here, I am defining the threshold as 50. Nevertheless, the parameter can be changed.

frequent_notes = [note_ for note_, count in freq.items() if count>=50]
print(len(frequent_notes))


#!/usr/bin/env python3
import matplotlib
matplotlib.use('Agg') # to work without X
import matplotlib.pyplot as plt
import numpy as np
import os

from moviepy.editor import VideoClip
from moviepy.video.io.bindings import mplfig_to_npimage

params = {
#"figure.figsize" :  [3.36, 2.976],  # figure size in inches
#                       "figure.dpi"     :  300,       # figure dots per inch
#                       "font.size"      :  8,       # this one acutally changes tick labels
#                       "font.family"    : "Times New Roman",
#                      'svg.fonttype'    : 'none',  # plot text as text - not paths or clones or other nonsense
#                       "legend.fontsize": "large",   # these don't seem to do anything for the tick label font size
#                       "axes.titlesize" : "large",
#                       "xtick.labelsize": "large",
                       "ytick.labelsize": 20, # for colorbar ticks
                       "axes.labelsize" : 20, # for colorbar labels
}

plt.rcParams.update(params)

# find time index of the time closest to t
def find_time_index(times, t):
    ti = np.abs((times[:]-t)).argmin()
    return ti

class Plot:
    def __init__(self, data, outdir=None, size=1080, colorbar=False, time_fmt=None, text=""):
        self.outdir = outdir
        self.colorbar = colorbar
        self.time_fmt = time_fmt
        self.data = data
        self.text = text # optional text instead of time stamp

        dpi = 80
        sizex = size
        if colorbar:
            sizex *= 4/3
        self.fig = plt.figure(figsize=(sizex/dpi, size/dpi), dpi=dpi)

        self.timetext = plt.text(.98, .98, "",  #y was 0.05
                                 horizontalalignment='right',
                                 verticalalignment='top',
                                 #transform=fig.gca().transAxes,
                                 transform=plt.gcf().transFigure,
                                 color='#ffbbcc',
                                 size=30)
        plt.axis('off')

    def format_time(self, t):
        hours = int(t/3600)
        minutes = int(t/60)%60
        seconds = int(t)%60
        if self.time_fmt == 'hms':
            return "%3.0f:%02d:%02d"%(hours,minutes,seconds)
        return "%3.0f h"%(hours)

    # times is a list of time points to plot, in hours
    def plot(self, times=[24], run=None, filename=None):
        for t in times:
            t_sec = t*3600
            ti = find_time_index(self.data.time, t_sec)
            if np.abs(self.data.time[ti]-t_sec) < 600:
                self.select_time(ti, run)
                if not filename:
                    f=self.plotname+'%02d.png'%t
                else:
                    f=filename
                if run:
                    f = run + '_' + f
                outfile=os.path.join(self.outdir, f)
                self.fig.savefig(outfile)

    def movie(self, fps=24, run='', vx=0, vy=0):
        nframes = self.data.time.shape[0]
        duration= nframes/fps

        def make_frame(t):
            nonlocal self, fps, run
            ti = int(t*fps+.5)
            self.select_time(ti, run, vx=vx, vy=vy)
            return mplfig_to_npimage(self.fig)

        animation = VideoClip(make_frame, duration=duration)
        outfile=os.path.join(self.outdir, self.moviename)
        animation.write_videofile(outfile, fps=fps)


import argparse
import os
import re
import yaml

# Workaround for unknown yaml tags
def yaml_default_ctor(loader, tag_suffix, node):
    return tag_suffix + ' ' + node.value

yaml.add_multi_constructor('', yaml_default_ctor)

class PlantUmlNode():
    def __init__(self, node_dn):
        self._dn = node_dn
        self.attributes = []
        self.operations = []
        self.extras = []

    def write(self, f):
        f.write('class {0} {{\n'.format(self._dn))
        f.write('-- Properties --\n')
        for item in self.attributes:
            f.write('<{type}> {name}: {contract}\n'.format(**item))
        f.write('-- Workflows --\n')
        for item in self.operations:
            f.write('{name}()\n'.format(**item))
        f.write('-- Namespaces --\n')
        for item in self.extras:
            if item['key'] != '=':
                f.write('{key}: {value}\n'.format(**item))
        f.write('}\n')

    def add_attribute(self, name, type, contract):
        d = {'type': type, 'name': name, 'contract': contract}
        self.attributes.append(d)

    def add_operation(self, name):
        d = {'name': name}
        self.operations.append(d)

    def add_extra(self, key, value):
        d = {'key': key, 'value': value}
        self.extras.append(d)

class DslSpec():
    def __init__(self, class_dn, basepath='.'):
        self._dn = class_dn
        self._id = self._dn.replace('.', '_')
        self._name = self._dn.split('.')[-1]
        self._ns = ('=', self._dn.split('.')[:-1])
        self._graph = None
        self.is_virtual = True
        self.parent_dn = None
        self.manifest = None

        manifest_path = os.path.join(basepath, self._dn, 'manifest.yaml')
        if os.path.exists(manifest_path):
            self.manifest = yaml.load(open(manifest_path))

        if self.manifest:
            self.is_virtual = False
            self.name = self.manifest['Name']
            if 'Extends' in self.manifest:
                self.parent_dn = self.get_dn(self.manifest['Extends'])

    def split_ns(self, name=None):
        if name:
            parts = name.split(':')
            if len(parts) == 1:
                parts.insert(0, '=')
        else:
            parts = ['=', self._name]
        return parts

    def get_ns(self, name=None):
        parts = self.split_ns(name)
        return {
            'key': parts[0],
            'value': self.manifest['Namespaces'][parts[0]]
        }

    def get_dn(self, name=None):
        parts = self.split_ns(name)
        parts[0] = self.get_ns(parts[0] + ':')['value']
        return '.'.join(parts)

    def get_name(self):
        return self._name

class DslPlantUmlNode(DslSpec):
    def write(self, file):
        uml_class = PlantUmlNode(self._dn)
        ext_classes = []
        if not self.is_virtual:
            namespaces = [self.get_ns()]
            for name, item in self.manifest.get('Properties', {}).iteritems():
                item_type = item.get('Type', 'In')
                item_contract = str(item.get('Contract', 'UNDEFINED'))
                match = re.search('class\((.*?)\)', item_contract)
                if match:
                    ns = self.get_ns(match.group(1))
                    ext_classes.append(self.get_dn(match.group(1)))
                    if not ns in namespaces:
                        namespaces.append(ns)
                uml_class.add_attribute(
                    name,
                    type=item_type,
                    contract=item_contract
                )
            for m in self.manifest.get('Workflow', []):
                uml_class.add_operation(m)
            for ns in namespaces:
                uml_class.add_extra(ns['key'], ns['value'])
        uml_class.write(file)
        return ext_classes

class DslPlantUmlGraph():
    def __init__(self):
        self._nodes = []
        self._edges = []
        self._options = {}
        self._file = None

    def write(self, classname, level=0):
        if level == 0:
            self._file.write('@startuml\n')
            if self.get_option('NoNamespaces', False):
                self._file.write('set namespaceSeparator none\n')

        if self.node_exists(classname):
            return

        self.add_node(classname)

        node = DslPlantUmlNode(classname)
        ext_classes = node.write(self._file)

        if not self.get_option('ParentsOnly', False):
            for ext_class in ext_classes:
                self.add_edge(
                    from_node=ext_class,
                    to_node=classname,
                    edge_type='<..'
                )
                self.write(ext_class, level + 1)

        if node.is_virtual:
            return

        if node.parent_dn:
            self.add_edge(
                from_node=node.parent_dn,
                to_node=classname,
                edge_type='<|--'
            )
            self.write(node.parent_dn, level + 1)

        if level == 0:
            for edge in self._edges:
                self._file.write('{from_node} {type} {to_node}\n'
                    .format(**edge))
            self._file.write('@enduml\n')
            self._file.close()

    def add_node(self, classname):
        self._nodes.append(classname)

    def node_exists(self, dn):
        return dn in self._nodes

    def add_edge(self, from_node, to_node, edge_type):
        edge = {'from_node': from_node, 'to_node': to_node, 'type': edge_type}
        if not edge in self._edges:
            self._edges.append(edge)

    def set_option(self, key, value):
        self._options[key] = value

    def get_option(self, key, default=None):
        return self._options.get(key, default)

    def open_file(self, file_name):
        self._file = open(file_name, 'w')

    def close_file(self):
        self._file.close()

parser = argparse.ArgumentParser(description='Qwerty')
parser.add_argument('classname',
                    default='com.mirantis.murano.demoApp.DemoHost',
                    help='Dsl Class Name to draw.', nargs='?')
parser.add_argument('-n', '--no-namespaces', action='store_true')
parser.add_argument('-p', '--parents-only', action='store_true')
args = parser.parse_args()

graph = DslPlantUmlGraph()
graph.set_option('NoNamespaces', args.no_namespaces)
graph.set_option('ParentsOnly', args.parents_only)
graph.open_file('plantuml.txt')
graph.write(args.classname)
graph.close_file()

os.system('java -jar plantuml.jar plantuml.txt')
os.system('xdg-open plantuml.png')


import pytest

import tempfile

import numpy as np
import pandas as pd

from e15190.runlog import query

class TestElogQuery:
    def test___init__(self):
        q = query.ElogQuery(
            load_run_batches=False,
            update_run_batches=False,
            save_run_batches=False,
        )
        assert isinstance(q, query.ElogQuery)
        assert isinstance(q.df, pd.DataFrame)
        assert isinstance(q.batch_unified_properties, list)
        assert isinstance(q.max_run_gap, np.timedelta64)

    def test_determine_run_batches(self):
        q = query.ElogQuery(
            load_run_batches=False,
            update_run_batches=False,
            save_run_batches=False,
        )
        assert q.df.index.names == [None]

        q.determine_run_batches()
        assert q.df.index.names == ['ibatch', 'irun']

        ibatches = list(q.df.index.get_level_values('ibatch').unique())
        assert ibatches == list(range(len(ibatches)))

        iruns = list(q.df.index.get_level_values('irun').unique())
        assert iruns == list(range(len(iruns)))

        for ibatch in range(len(ibatches)):
            df_batch = q.df.loc[ibatch]

            # check uniqueness of unified properties
            for property in q.batch_unified_properties:
                assert len(df_batch[property].unique()) == 1

            # check maximum run gap
            for i in range(len(df_batch) - 1):
                assert df_batch['begin_time'].iloc[i + 1] - df_batch['end_time'].iloc[i] <= q.max_run_gap

    def test_get_run_batches_summary(self):
        q = query.ElogQuery(
            load_run_batches=False,
            update_run_batches=False,
            save_run_batches=False,
        )
        q.determine_run_batches()
        assert q.df_batches is None

        df_batches = q.get_run_batches_summary()
        assert df_batches.equals(q.df_batches)
        assert isinstance(df_batches, pd.DataFrame)
        assert df_batches.index.names == ['ibatch']
        assert len(df_batches) == len(q.df.index.get_level_values('ibatch').unique())

    def test_save_run_batches(self):
        q = query.ElogQuery(
            load_run_batches=False,
            update_run_batches=True,
            save_run_batches=False,
        )

        path = tempfile.NamedTemporaryFile(suffix='.csv').name
        q.save_run_batches(path)

        df_batches = pd.read_csv(path)
        assert len(df_batches) == len(q.df.index.get_level_values('ibatch').unique())
        assert df_batches['ibatch'].to_list() == list(range(len(df_batches)))

    def test_load_run_batches(self):
        q = query.ElogQuery(
            load_run_batches=False,
            update_run_batches=False,
            save_run_batches=False,
        )

        assert q.df_batches is None
        q.load_run_batches()
        assert isinstance(q.df_batches, pd.DataFrame)
        assert q.df_batches['run_min'].dtype == int
        assert q.df_batches['trigger_rate_mean'].dtype == float
        assert 'datetime' in str(q.df_batches['begin_time'].dtype).lower()
        assert 'datetime' in str(q.df_batches['end_time'].dtype).lower()
        assert 'timedelta' in str(q.df_batches['total_elapse'].dtype).lower()
        assert 'timedelta' in str(q.df_batches['total_gap_time'].dtype).lower()

@pytest.fixture
def run_2200():
    return dict(
        run=2200,
        begin_time=pd.Timestamp('2018-02-13 18:09:02'),
        end_time=pd.Timestamp('2018-02-13 18:36:39'),
        elapse=pd.Timedelta('0 days 00:27:37'),
        target='Ni58',
        beam='Ca40',
        beam_energy=140.0,
        shadow_bar='out',
        trigger_rate=2112.0,
    )

@pytest.fixture
def run_4100():
    return dict(
        run=4100,
        begin_time=pd.Timestamp('2018-03-11 03:40:40'),
        end_time=pd.Timestamp('2018-03-11 04:12:19'),
        elapse=pd.Timedelta('0 days 00:31:39'),
        target='Ni64',
        beam='Ca48',
        beam_energy=140.0,
        shadow_bar='in',
        trigger_rate=2463.0,
    )

class TestQuery:
    def test_get_ibatch(self):
        ibatch = query.Query.get_ibatch(2200)
        assert ibatch == 3

        ibatch = query.Query.get_ibatch(4100)
        assert ibatch == 33

    def test__get_run_query(self, run_2200, run_4100):
        entry = query.Query._get_run_query(2200)
        assert isinstance(entry, pd.Series)
        for key, value in run_2200.items():
            assert entry[key] == value
        assert isinstance(entry['comment'], str)

        entry = query.Query._get_run_query(4100)
        assert isinstance(entry, pd.Series)
        for key, value in run_4100.items():
            assert entry[key] == value
        assert isinstance(entry['comment'], str)

    def test_get_run_info(run, run_2200, run_4100):
        info = query.Query.get_run_info(2200)
        assert isinstance(info, dict)
        for key, value in run_2200.items():
            assert info[key] == value
        assert isinstance(info['comment'], str)
        assert 'ibatch' in info
        info = query.Query.get_run_info(2200, include_ibatch=False)
        assert 'ibatch' not in info

        info = query.Query.get_run_info(4100)
        assert isinstance(info, dict)
        for key, value in run_4100.items():
            assert info[key] == value
        assert isinstance(info['comment'], str)
        assert 'ibatch' in info
        info = query.Query.get_run_info(4100, include_ibatch=False)
        assert 'ibatch' not in info

    def test__get_batch_query(self):
        for ibatch in query.Query.elog.df_batches.index.get_level_values('ibatch'):
            entry = query.Query._get_batch_query(ibatch)
            assert isinstance(entry, pd.DataFrame)
            assert entry.index.names == ['irun']

    def test_get_batch(self):
        for ibatch in query.Query.elog.df_batches.index.get_level_values('ibatch'):
            batch = query.Query.get_batch(ibatch)
            assert isinstance(batch, pd.DataFrame)
            assert batch.index.names == [None]
            assert batch.index.values.tolist() == list(range(len(batch)))

    def test_get_batch_runs(self):
        runs = query.Query.get_batch_runs(2)
        assert set(runs) == {2142, 2143, 2144, 2145, 2146, 2147, 2150, 2151, 2152}

    def test_get_batch_info(self):
        for ibatch in query.Query.elog.df_batches.index.get_level_values('ibatch'):
            info = query.Query.get_batch_info(ibatch)
            assert isinstance(info, dict)
            assert info['ibatch'] == ibatch
            assert isinstance(info['comment'], (str, list))
            assert 'runs' in info

            info = query.Query.get_batch_info(ibatch, include_comments=False)
            assert info['comment'] is None

            info = query.Query.get_batch_info(ibatch, include_runs=False)
            assert 'runs' not in info

    def test_get_n_batches(self):
        assert query.Query.get_n_batches() == len(query.Query.elog.df_batches)
        assert query.Query.get_n_batches() >= 60

    def test_targets(self):
        assert set(query.Query.targets()) == set(['Ni58', 'Ni64', 'Sn112', 'Sn124'])

    def test_beams(self):
        assert set(query.Query.beams()) == set(['Ca40', 'Ca48'])

    def test_beam_energies(self):
        assert set(query.Query.beam_energies()) == set([56.0, 140.0])

    def test_select_runs(self):
        # example 1
        runs = query.Query.select_runs('target == "Ni58"')
        assert 2134 in runs
        assert 2135 in runs
        assert 2136 not in runs
        assert 2142 in runs
        assert 4617 in runs
        assert 4618 in runs
        assert 4619 in runs
        assert min(runs) == runs[0] == 2134
        assert max(runs) == runs[-1] == 4619

        # example 2
        runs = query.Query.select_runs(
            ' & '.join([
                'beam == "Ca40"',
                'target == "Ni64"',
                'beam_energy == 56.0',
                'shadow_bar == "in"',
            ])
        )
        expected_runs = list(range(2512, 2523 + 1))
        expected_runs.remove(2516)
        assert runs == expected_runs

        # example 3
        runs = query.Query.select_runs(
            'beam == "Ca48" & beam_energy == 56',
            comment_cut='mb singles 301',
            case=False,
        )
        assert runs[:3] == [4593, 4594, 4595]
        assert runs[-3:] == [4659, 4660, 4661]

    def test_select_batches(self):
        ibatches = query.Query.select_batches('n_runs > 35')
        assert ibatches == [22, 33, 49]


import h5py
import argparse
from scipy import constants
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy.optimize import curve_fit

def linear(x, m, n):
    return m * x + n

def main(raw_args=None):
    # create an axis
    parser = argparse.ArgumentParser(
        description="Calculate virtual distance from peak positions list, using reflections index and unit cell parameter for Au powder patterns."
    )
    parser.add_argument(
        "-i",
        "--input",
        type=str,
        action="store",
        help="hdf5 input image in which peak positions is stored",
    )

    args = parser.parse_args(raw_args)

    beam_energy = 5.86 * 1e-13
    px_size = 75 * 1e-6
    _lambda = (
        1e10
        * constants.h
        * constants.c
        / math.sqrt(
            (beam_energy) ** 2
            + (2 * beam_energy * constants.electron_mass * (constants.c**2))
        )
    )
    print(_lambda)
    # reflections_list=[[1,1,1],[0,0,2],[0,2,2],[1,1,3],[2,2,2],[0,0,4],[1,3,3],[0,2,4]]
    reflections_list = [[1, 1, 1], [0, 0, 2], [0, 2, 2], [1, 1, 3]]
    a = 407.3 * 1e-2

    q_t = []
    for idx, i in enumerate(reflections_list):
        q_hkl = math.sqrt(i[0] ** 2 + i[1] ** 2 + i[2] ** 2) / a
        q_t.append(q_hkl)
    theta_t = [2 * np.arcsin(k * _lambda / 2) for k in q_t]
    print("q_t", q_t)
    print("theta_t", theta_t)

    peak_input = [
        90,
        105,
        146,
        172,
    ]  # Uncomment next line for peak positions from input file

    # x_o=np.array(hf['peak_position'])*px_size
    x_o = np.array(peak_input) * px_size

    tan_theta_o = np.tan(theta_t)

    x = tan_theta_o
    print("tan_o", x)
    a = np.ones((len(x), 2))
    for idx, i in enumerate(x):
        a[idx][0] = i

    y = x_o
    rough_fit = np.linalg.lstsq(a, y, rcond=None)
    m0, n0 = rough_fit[0]
    popt, pcov = curve_fit(linear, x, y, p0=[m0, 0])
    residuals = y - linear(x, *popt)
    ss_res = np.sum(residuals**2)
    ss_tot = np.sum((y - np.mean(y)) ** 2)
    r_squared = 1 - (ss_res / ss_tot)
    print("d", popt)
    print("R²", r_squared)
    x_fit = np.arange(0.0, 0.004, 0.0005)
    plt.scatter(x, y, marker="o")
    plt.plot(x_fit, linear(x_fit, *popt), "r:", label="fit")

    plt.xlabel("tanθ (rad)")
    plt.ylabel("Peak radius (m)")
    plt.show()

if __name__ == "__main__":
    main()


import pickle
import torch
from torch.autograd import Variable
from build_vocab import Vocab
from data_loader import get_data_loader
from data_loader import get_styled_data_loader
from models import EncoderCNN
from models import FactoredLSTM
from loss import masked_cross_entropy

def main():
    with open("data/vocab.pkl", 'rb') as f:
        vocab = pickle.load(f)

    img_path = "data/flickr7k_images"
    cap_path = "data/factual_train.txt"
    styled_path = "data/humor/funny_train.txt"
    data_loader = get_data_loader(img_path, cap_path, vocab, 3)
    styled_data_loader = get_styled_data_loader(styled_path, vocab, 3)

    encoder = EncoderCNN(30)
    decoder = FactoredLSTM(30, 40, 40, len(vocab))

    if torch.cuda.is_available():
        encoder = encoder.cuda()
        decoder = decoder.cuda()

    # for i, (images, captions, lengths) in enumerate(data_loader):
    for i, (captions, lengths) in enumerate(styled_data_loader):
        # images = Variable(images, volatile=True)
        captions = Variable(captions.long())

        if torch.cuda.is_available():
            # images = images.cuda()
            captions = captions.cuda()

        # features = encoder(images)

        outputs = decoder(captions, features=None, mode="humorous")
        print(lengths - 1)
        print(outputs)
        print(captions[:, 1:])

        loss = masked_cross_entropy(outputs, captions[:, 1:].contiguous(), lengths - 1)

        print(loss)

        break

if __name__ == '__main__':
    main()


import setuptools

def get_requires(filename):
    requirements = []
    with open(filename, "rt") as req_file:
        for line in req_file.read().splitlines():
            if not line.strip().startswith("#"):
                requirements.append(line)
    return requirements

project_requirements = get_requires("requirements.txt")
setuptools.setup(
    name="snake",
    version="2.2.2",
    install_requires=project_requirements
)


'''
Approach
    - 길이가 1~7이하 문자열

>>> 9999999 이하의 소수를 모두 찾아본다

'''
from itertools import permutations
import math

def findPrime(limit):
    allNum = [True]*(limit+1)
    allNum[0] = allNum[1] = False
    # 제곱근까지만 계산해서 소수인지 확인
    for i in range(2, int(math.sqrt(limit)+1)):
        # i의 배수를 모두 false처리
        if allNum[i]:
            j = 2
            while i*j<=limit:
                allNum[i*j] = False
                j +=1
    # true인 수는 모두 프라임이다
    return allNum

def solution(numbers):
    answer = 0
    numbers = list(numbers)
    limit = 10**len(numbers)

    # limit까지의 수 중에서 prime 수를 찾음
    prime = findPrime(limit)

    # 소수 완탐
    visited = set()
    for i in range(1, len(numbers)+1):
        perm = set(permutations(numbers,i))
        for c in perm:
            c = int(''.join(c))
            if prime[c] and c not in visited:
                print(c)
                visited.add(c)
                answer +=1
    # 완탐으로 푸는법
    # for i in range(2, limit):
    #     flag = True
    #     temp = numbers[:]
    #     if prime[i]:
    #         for s in str(i):
    #             if s not in temp:
    #                 flag = False
    #                 break
    #             temp.remove(s)
    #         if flag:
    #             answer +=1

    return answer

프로그래머스/lv2/42839. 소수 찾기/소수 찾기.py


import os
import pkg_resources
import yaml

from fget.utils import fgetprint
from fget.resource.root import Root

class CachedSettings(object):

    def __init__(self, cache_dir):
        self.cache_dir = cache_dir

    def init(self):

        settings_filename = 'fget.yaml'
        cached_filename = 'fget.jobs'

        cached_settings_file = os.path.join(self.cache_dir, cached_filename)

        self.cached_settings = {}
        if not os.path.isfile(cached_settings_file):

            fgetprint('Initiating. Please wait...')

            settings_file = \
                pkg_resources.resource_filename('fget', settings_filename)

            with open(settings_file) as f:
                settings = yaml.load(f.read())

            for url in settings.get('JENKINS_URLS', []):
                url = url.strip('/')
                fgetprint('Retrieving jobs from {0}'.format(url))
                root_resource = Root(url)
                for job in root_resource.get_jobs():
                    if url not in self.cached_settings:
                        self.cached_settings[url] = []
                    self.cached_settings[url].append(str(job['name']))
            with open(cached_settings_file, 'w') as f:
                for key in self.cached_settings.keys():
                    f.write(key + '\n')
                    for value in self.cached_settings[key]:
                        f.write(value + '\n')

            fgetprint('Initiating. Finished.')
        else:
            with open(cached_settings_file) as f:
                for line in f:
                    if line.startswith('http://'):
                        url = line.strip()
                        self.cached_settings[url] = []
                        continue
                    self.cached_settings[url].append(line.strip())

    def get_settings(self):
        return self.cached_settings