# -*- coding: utf-8 -*-
"""
Created on Tue Nov  8 12:58:23 2016

@author: MIKI yuya
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import animation
plt.rcParams['animation.ffmpeg_path'] = 'C:\\ffmpeg-20161122-d316b21-win64-static\\bin\\ffmpeg.exe'
# def function
def update_quiver(i):
    global x,z,Su,Sz,numt,M
    if numt <3000:
        numt=numt+1
    else:
        numt=3000
    pX=[]
    pZ=[]
    pU=[]
    pW=[]
    for numx in range(len(x)):
        for numz in range(len(z)):
            pX.append(x[numx])
            pZ.append(z[numz])
            pU.append(Su[numx,numt,numz])
            pW.append(Sw[numx,numt,numz])
    M = np.sqrt(np.power(pU, 2) + np.power(pW, 2))
    Q.set_UVC(pU,pW,M)
    return Q
##############################################################################################
num_data=2814
row1=28
row2=num_data
skip1=5
skip2=skip1+row1
skip3=skip2+num_data
f1=pd.read_csv('HISTORY5_movie.txt',header=None, skiprows=skip1, nrows=row1, sep='#', skipinitialspace=True)
f2=pd.read_csv('HISTORY5_movie.txt',header=None, skiprows=skip2, nrows=row2, skipinitialspace=True,delim_whitespace=True)
f3=pd.read_csv('HISTORY5_movie.txt',header=None, skiprows=skip3, skipinitialspace=True,delim_whitespace=True)
f3.index=list(f3.iloc[:,0])
# default data set
header1=[f1.iloc[:,1]]
d_head1=pd.DataFrame(header1)
data1=f3.iloc[:,0:row1]
data1.columns=list(header1)

# User define data set

header2=[f2.iloc[:,1],f2.iloc[:,4],f2.iloc[:,6],f2.iloc[:,8]]
d_head2=pd.DataFrame(header2)
d_head2.index=['Define','x','y','z']
data2=f3.iloc[:,row1:len(f3.columns)]
data2.columns=[i for i in range(len(data2.columns))]
#usr_data=d_head2.append(data2)
data=np.array(data2)
del f1,f2,f3,header1,header2,d_head1,d_head2,
del num_data,row1,row2,skip1,skip2,skip3
del data1,data2

# making dataset END
##############################################################################################
# make x list
x=np.arange(0.999,1,0.00005)
# make z list
z_max=0.00176
dz=(8E-5)/3
z_list=[]
z_start=0
zn=z_start
for nz in range(0,int((z_max-z_start)/dz)+1):
    z_list.append(zn)
    zn=round(zn+dz,8)
z_list[nz]=z_max
z=np.array(z_list)
del z_start,zn,z_max,z_list,dz,nz
# make Uave & Vave
val=np.hsplit(data,42)
lenv=int(len(val)/2)
ulist=[]
wlist=[]
for i in range(lenv):
#    print(np.mean(val[i]))
    ulist.append((val[i]))
    wlist.append((val[i+lenv]))
U=np.array(ulist)
W=np.array(wlist)
####################################################################################
nu=2.0E-7
myu=1.21*nu
#shear
U2=np.multiply(myu,U)
W2=np.multiply(myu,W)
# Shear Stress
Su=np.divide(U2,1E-5)
Sw=np.divide(W2,1E-5)

del nu,myu,U,W,U2,W2,ulist,wlist,val,lenv,i
####################################################################################
# making shear stress animation & Cf-Rex plot
# animation :(x,y,z,Stress(U,W))
# plot is vector fuild
# using plt.quiver
# helpful below
# http://stackoverflow.com/questions/19329039/plotting-animated-quivers-in-python
####################################################################################

pX=[]
pZ=[]
pU=[]
pW=[]
numt=0
for numx in range(len(x)):
    for numz in range(len(z)):
        pX.append(x[numx])
        pZ.append(z[numz])
        pU.append(Su[numx,numt,numz])
        pW.append(Sw[numx,numt,numz])
    M = np.sqrt(np.power(pU, 2) + np.power(pW, 2))
#    im=plt.quiver(pX,pZ,pU,pW,M,angles='xy')

fig, ax = plt.subplots(figsize=(10,7.5))
Q=ax.quiver(pX,pZ,pU,pW,M,angles='xy')
ax.set_xlim(0.9990, 1)
ax.set_ylim(0, 0.0018)
ax.set_title('Shear Stress')
ax.set_xlabel('x')
ax.set_ylabel('y')

anim = animation.FuncAnimation(fig, update_quiver,frames=2999)
#plt.show()
FFwriter = animation.FFMpegWriter(fps=60)
anim.save('./hoge.mp4', writer = FFwriter, dpi=300)
#anim.save('hoge1.gif', writer='imagemagick', fps=45, dpi=100)


#!/usr/bin/env python
# ==============================================================================
#          \file   inference.py
#        \author   chenghuige
#          \date   2017-09-14 07:45:47.415075
#   \Description
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys, os
from deepiu.util.text_predictor import TextPredictor
from deepiu.util.sim_predictor import SimPredictor
from deepiu.util import ids2text, text2ids
import melt, gezi
import numpy as np
import traceback
#FIXME ValueError: At least two variables have the same name: InceptionResnetV2/Repeat/block35_9/Conv2d_1x1/biases
try:
  import conf
  from conf import TEXT_MAX_WORDS
except Exception:
  from deepiu.image_caption.conf import TEXT_MAX_WORDS

image_dir = '/home/gezi/data2/data/ai_challenger/image_caption/pic/'
image_file = '6275b5349168ac3fab6a493c509301d023cf39d3.jpg'

image_model_checkpoint_path = '/home/gezi/data/image_model_check_point/inception_resnet_v2_2016_08_30.ckpt'
model_dir = '/home/gezi/new/temp/image-caption/ai-challenger/model/showattentell/'
sim_model_dir = '/home/gezi/new/temp/image-caption/ai-challenger/model/bow/'
vocab_path = '/home/gezi/new/temp/image-caption/ai-challenger/tfrecord/seq-basic/vocab.txt'
valid_dir = '/home/gezi/new/temp/image-caption/ai-challenger/tfrecord/seq-basic/valid'

image_model_name = 'InceptionResnetV2'

feature_name = melt.image.get_features_name(image_model_name)

#if finetuned model, just  TextPredictor(model_dir, vocab_path)
if not melt.varname_in_checkpoint(image_model_name, model_dir):
  predictor = TextPredictor(model_dir, vocab_path, image_model_checkpoint_path, image_model_name=image_model_name, feature_name=feature_name)
else:
  predictor = TextPredictor(model_dir, vocab_path)

vocab = ids2text.vocab

text2ids.init(vocab_path)

sim_predictor = SimPredictor(sim_model_dir, image_model_checkpoint_path, image_model_name=image_model_name, index=-1)

text_strs = np.load(os.path.join(valid_dir, 'distinct_text_strs.npy'))
img2text = np.load(os.path.join(valid_dir, 'img2text.npy')).item()

while True:
  image_file = raw_input('image_file like 6275b5349168ac3fab6a493c509301d023cf39d3.jpg:')
  if image_file == 'q':
    break

  image_path = os.path.join(image_dir, image_file)
  print('image_path:', image_path)

  if not os.path.exists(image_path):
    print('image path not find!')
    continue

  try:
    hits = img2text[image_file]
    texts = [text_strs[hit] for hit in hits]
    for text in texts:
      word_ids = text2ids.text2ids(text)
      seg_text = text2ids.ids2text(word_ids, print_end=False)
      print('label:', text, seg_text)
      words_importance = sim_predictor.words_importance([word_ids])
      words_importance = words_importance[0]
      print('word importance:')
      for i in range(len(word_ids)):
        if word_ids[i] == 0:
          break
        print(vocab.key(int(word_ids[i])), words_importance[i], end='|')
      print()
  except Exception:
    print(traceback.format_exc(), file=sys.stderr)
    pass

  image = melt.read_image(image_path)
  word_ids, scores = predictor.word_ids([image])
  word_id = word_ids[0]
  score = scores[0]
  print('best predict:', ids2text.translate(word_id[0]),  score[0], '/'.join([vocab.key(int(id)) for id in word_id[0] if id != vocab.end_id()]))

  l = [id for id in word_id[0] if id != vocab.end_id()]
  l = gezi.pad(l, TEXT_MAX_WORDS)
  words_importance = sim_predictor.words_importance([l])
  words_importance = words_importance[0]

  print('word importance:')
  for i in range(len(word_id[0])):
    if word_id[0][i] == vocab.end_id():
      break
    print(vocab.key(int(word_id[0][i])), words_importance[i], end='|')
  print()

  for i in range(len(word_id)):
    if i > 0:
      print('top%d predict:'%i, ids2text.translate(word_id[i]),  score[i], '/'.join([vocab.key(int(id)) for id in word_id[i] if id != vocab.end_id()]))
      l = [id for id in word_id[i] if id != vocab.end_id()]
      l = gezi.pad(l, TEXT_MAX_WORDS)
      words_importance = sim_predictor.words_importance([l])
      words_importance = words_importance[0]

      print('word importance:')
      for j in range(len(word_id[i])):
        if word_id[i][j] == vocab.end_id():
          break
        print(vocab.key(int(word_id[i][j])), words_importance[j], end='|')
      print()

  scores, word_ids = sim_predictor.top_words([image], 50)
  scores = scores[0]
  word_ids = word_ids[0]
  print('topwords of image:')
  for word_id, score in zip(word_ids, scores):
    print(vocab.key(int(word_id)), score, end='|')
  print()


import pygame
from Beonho import Beonho

class Board(object):

    def __init__(self,window):
        self.window = window
        self.side = window[1]-100
        if self.side%9 != 0:
            self.side -= self.side%9
        self.square = self.side//9
        self.x = self.window[0]//2 - self.side//2
        self.y = 50
        #self.beonho = [1,2,3,4,5,6,7,8,9]
        self.beonho = Beonho()
        self.selected = [9,9] #coords of square from 0-8, starts outside so is blank

    def update(self,num):
        #spaghetti code, seems to process index backwards...idk why
        self.beonho[self.selected[1]][self.selected[0]] = num

    def draw_board(self,screen):
        pygame.draw.rect(screen, 0xFFFFFF, [self.x, self.y, self.side, self.side])
        pygame.draw.rect(screen,0x000000,[self.x,self.y,self.side,self.side],4)
        for i in range(1,9):
            #vertical line
            weight = 1
            if i%3==0:
                weight = 3
            pygame.draw.line(screen,0x0000000,[self.x+self.square*i,self.y],
                             [self.x+self.square*i,self.y+self.side],weight)
            #horizontal line
            pygame.draw.line(screen, 0x0000000, [self.x, self.y+self.square*i],
                             [self.x + self.side, self.y + self.square*i], weight)

        #draw highlighted square
        offset = 1
        if 0<=self.selected[0]<9 and 0<=self.selected[1]<9 :
            pygame.draw.rect(screen,0xFFFF00,[(self.selected[0])*self.square+self.x+offset,
                                          (self.selected[1])*self.square+self.y+offset,
                                          self.square-offset,self.square-offset])
        self.draw_rows(screen)

    def draw_rows(self,screen):
        size = self.square-2
        font = pygame.font.SysFont('serif', size)
        for i in range(0,9):
            for j in range(0,9):
                if self.beonho[i][j] != 0:
                    text = font.render(str(self.beonho[i][j]), True, (0x00, 0x00, 0x00))
                    screen.blit(text, [self.x+self.square*j+self.square//3,self.y-2+self.square*i])


from robocode_ls_core.robotframework_log import get_logger

log = get_logger(__name__)

class _Collector(object):
    def __init__(self, selection, token):
        from robotframework_ls.impl.string_matcher import RobotStringMatcher
        from robotframework_ls.impl.string_matcher import (
            build_matchers_with_resource_or_library_scope,
        )

        token_str = token.value

        self.completion_items = []
        self.selection = selection
        self.token = token

        self._matcher = RobotStringMatcher(token_str)
        self._scope_matchers = build_matchers_with_resource_or_library_scope(token_str)

    def accepts(self, keyword_name):
        if self._matcher.accepts_keyword_name(keyword_name):
            return True
        for matcher in self._scope_matchers:
            if matcher.accepts_keyword_name(keyword_name):
                return True
        return False

    def _create_completion_item_from_keyword(
        self, keyword_found, selection, token, col_delta=0
    ):
        """
        :param IKeywordFound keyword_found:
        :param selection:
        :param token:
        """
        from robocode_ls_core.lsp import (
            CompletionItem,
            InsertTextFormat,
            Position,
            Range,
            TextEdit,
        )
        from robocode_ls_core.lsp import MarkupKind

        label = keyword_found.keyword_name
        text = label

        for i, arg in enumerate(keyword_found.keyword_args):
            arg = arg.replace("$", "\\$").replace("{", "").replace("}", "")
            if arg.startswith("**"):
                arg = "&" + arg[2:]

            elif arg.startswith("*"):
                arg = "@" + arg[1:]

            colon_i = arg.rfind(":")
            equals_i = arg.rfind("=")
            if colon_i != -1 and equals_i != -1 and equals_i > colon_i:
                arg = arg[:colon_i] + arg[equals_i:]

            text += "    ${%s:%s}" % (i + 1, arg)

        text_edit = TextEdit(
            Range(
                start=Position(selection.line, token.col_offset + col_delta),
                end=Position(selection.line, token.end_col_offset),
            ),
            text,
        )

        # text_edit = None
        return CompletionItem(
            keyword_found.keyword_name,
            kind=keyword_found.completion_item_kind,
            text_edit=text_edit,
            documentation=keyword_found.docs,
            insertTextFormat=InsertTextFormat.Snippet,
            documentationFormat=(
                MarkupKind.Markdown
                if keyword_found.docs_format == "markdown"
                else MarkupKind.PlainText
            ),
        ).to_dict()

    def on_keyword(self, keyword_found):
        col_delta = 0
        if not self._matcher.accepts_keyword_name(keyword_found.keyword_name):
            for matcher in self._scope_matchers:
                if matcher.accepts_keyword(keyword_found):
                    # +1 for the dot
                    col_delta = len(matcher.resource_or_library_name) + 1
                    break
            else:
                return  # i.e.: don't add completion

        item = self._create_completion_item_from_keyword(
            keyword_found, self.selection, self.token, col_delta=col_delta
        )

        self.completion_items.append(item)

def complete(completion_context):
    """
    :param CompletionContext completion_context:
    """
    from robotframework_ls.impl.collect_keywords import collect_keywords
    from robotframework_ls.impl import ast_utils

    token_info = completion_context.get_current_token()
    if token_info is not None:
        token = ast_utils.get_keyword_name_token(token_info.node, token_info.token)
        if token is not None:
            collector = _Collector(completion_context.sel, token)
            collect_keywords(completion_context, collector)

            return collector.completion_items

    return []


import click
import csv
import re

GRADE = re.compile("^L([0-9]+)$")
CONTINUOUS_GRADE = re.compile("^L([0-9]+\.[0-9]+)$")
SEP_LABELLED_RANKED_LIST = ' '
DEFAULT_CUTOFFS = [1000]

class continuous_grade(dict):
    def __new__(self, max):
        self = dict.__new__(self)
        self.max = max
        return self

    def __missing__(self, level):
        grade = level + 1.0
        if grade <= 0 or self.max < grade:
            raise ValueError()
        return grade

    def index(self, grade):
        if grade <= 0 or self.max < grade:
            raise ValueError()
        return grade - 1.0

def read_grades(string):
    if 0 < len(string) and string[0] == ':':
        result = continuous_grade(max=float(string[1:]))
        return result
    result = _read_ints(string, ':', '-g')
    _validate_order(string, result, '-g')
    _validate_positive(string, result, '-g')
    return result

def read_stops(string):
    result = _read_ints(string, ':', '-s')
    _validate_order(string, result, '-s')
    _validate_positive(string, result, '-s')
    return result

def read_cutoffs(string):
    if len(string) == 0:
        return DEFAULT_CUTOFFS
    return _read_ints(string, ',', '--cutoffs')

def read_rel_file(f, sep):
    rows = csv.reader(f, delimiter=sep)
    result = {}
    for idx, row in enumerate(rows):
        if len(row) != 2:
            raise Exception(
                "The rel file contains an invalid line at Line %s" % (idx+1))
        did, grade = row
        if did in result:
            raise Exception(
                "The rel file contains different grades for the same document.")
        grade = _parse_grade(grade, idx)
        result[did] = grade
    f.close()
    return result

def read_labelled_ranked_list(f):
    result = []
    if f:
        stream = f
    else:
        import sys
        stream = sys.stdin
    for idx, line in enumerate(stream):
        ls = line.strip().split(SEP_LABELLED_RANKED_LIST)
        if len(ls) == 1:
            ls.append(None)
        if len(ls) > 0:
            did, grade = ls[:2]
            if grade is not None:
                grade = _parse_grade(grade, idx)
            result.append([did, grade] + ls[2:])
    stream.close()
    return result

def count_judged(maxrel, qrels):
    '''
    Read relevance assessments and return
    - the number of judged X-rel docs (including 0-rel=judged nonrel)
    - the total number of judged rel docs
    '''
    xrelnum, jrelnum = {}, 0
    xrelnum[0] = 0
    for grade in qrels.values():
        if grade not in xrelnum:
            xrelnum[grade] = 0
        xrelnum[grade] += 1
        if 0 < grade:
            jrelnum += 1
    return xrelnum, jrelnum

def output_labelled_ranked_list(j, truncate, qrels, sysdoclab):
    syslen = 0
    for idx, didgrade in enumerate(sysdoclab):
        if len(didgrade) != 2:
            raise Exception("Invalid line format at Line %s" % (idx+1))
        did, grade = didgrade
        if grade is not None:
            raise Exception("Already labelled at Line %s" % (idx+1))
        grade = qrels.get(did, None)
        if j and grade is None:
            pass # do not output documents without rel judge
        else:
            print(did + (" L" + str(grade) if grade is not None else ""))
            syslen += 1
        if truncate is not None and syslen >= truncate:
            break

def compute_validation(j, grades, stops, beta, gamma, logb, rbp,
    xrelnum, jrelnum):
    if len(stops) != len(grades):
        raise click.BadParameter("the size of '-s' must be the same as '-g'")
    if beta < 0:
        raise click.BadParameter("the value of '--beta' must be positive")
    if gamma < 0 or gamma > 1:
        raise click.BadParameter("the value of '--gamma' must range from 0 to 1")
    if logb < 0:
        raise click.BadParameter(
            "the value of '--logb' must be 0 (natural log) or more")
    if rbp < 0 or rbp > 1:
        raise click.BadParameter("the value of '--rbp' must range from 0 to 1")
    # TODO: bug report
    # THIS EXCEPTION SHOULD NOT BE RAISED
    #if j and xrelnum[0] == 0:
    #    raise Exception("No judged nonrel: bpref etc. not computable")
    if jrelnum == 0:
        raise Exception(
            "No relevance document found in the relevance assessment file")

def _read_ints(string, sep, name):
    '''
    Return a list of int values

    string: option value
    sep: separator character
    name: option name
    '''
    try:
        ints = [int(i) for i in string.split(sep)]
    except:
        raise click.BadParameter(
            "'%s' for option '%s'" % (string, name))
    return ints

def _validate_order(string, values, name):
    '''
    Validate values that must be in ascending order

    string: option value
    values: values to be in ascending order
    name: option name
    '''
    for small, large in zip(values[:-1], values[1:]):
        if small > large:
            raise click.BadParameter(
                "'%s' for option '%s' must be in ascending order" % (string, name))

def _validate_positive(string, values, name):
    '''
    Validate values that must be positive

    string: option value
    values: values to be positive
    name: option name
    '''
    if any([v < 0 for v in values]):
        raise click.BadParameter(
            "'%s' for option '%s' must be positive" % (string, name))

def _parse_grade(grade, idx):
    m = CONTINUOUS_GRADE.match(grade)
    if m:
        return float(m.group(1))
    m = GRADE.match(grade)
    if m:
        return int(m.group(1))
    else:
        raise Exception(
            "An invalid grade at Line %s" % (idx+1))


"""
Performs xml parsing, collecting results within the object
"""

import xml.etree.ElementTree as etree
from operator import itemgetter
from datetime import datetime, timedelta
import copy
import time

import date_funcs

class xmlParser:
    def __init__(self, SMS_XML_FILE):
        self.SMS_XML_FILE = SMS_XML_FILE

        self.sms_count = 0

        self.person_info_schema = {"total" : 0, "num_recv" : 0, "num_sent" : 0,
                                   "msgs" : [],
                                   "recv_msgs_by_day" : [], "sent_msgs_by_day" : [], "date_range" : [],
                                   "recv_word_count_sorted" : [], "sent_word_count_sorted" : [],
                                   "recv_dates_sorted" : [], "sent_dates_sorted" : [], "dates_sorted" : [],
                                   "avg_response_time_me" : 0, "avg_response_time_them" : 0}

        self.person_info = {}
        self.person_info["me"] = copy.deepcopy(self.person_info_schema)

        self.ignored_words = ['i', 'to', 'you', 'and', 'the', '', 'a', 'u',
                              'my', 'of', 'it', "i'm", 'so', 'just', "it's", 'get',
                              'for', 'at', 'that', 'have', 'me', 'is', 'your',
                              'be', 'but', 'in', 'do', 'if', 'up', 'are', 'like',
                              'was', 'with', 'can', 'know', "i'll", "on", "really",
                              'this', 'not', "im", "there", "#55357;#56856;", "that's",
                              "all", "got", "been"]

    def run(self):
        self.read_xml()
        self.build_persons_metrics()

    def read_xml(self):
        print("Beginning XML Parse...")
        start_time = datetime.now()

        for __, elem in etree.iterparse(self.SMS_XML_FILE, events=('start', 'end', 'start-ns', 'end-ns')):
            sms_info = elem.attrib

            try:
                person = sms_info['contact_name']
                body = sms_info['body'].lower()
                type_ = sms_info['type'] # 1 = recv, 2 = sent
                date = sms_info['date']

                self.sms_count += 1

                if person in self.person_info:
                    self.add_to_person_info(person, body, type_, date)
                else:
                    self.person_info[person] = copy.deepcopy(self.person_info_schema)
                    self.add_to_person_info(person, body, type_, date)

            except:
                pass

        time_elapsed = datetime.now() - start_time
        print('XML Parse Time elapsed (hh:mm:ss.ms) {}'.format(time_elapsed))

    def build_persons_metrics(self):
        print("Building Person Metrics...")

        for person in self.person_info:
            self.sort_person_most_used_words(person)
            self.sort_msg_dates(person)
            self.get_avg_response_times(person)
            self.get_messages_per_day(person)

    def add_to_person_info(self, person, body, type_, date):
        self.person_info[person]["total"] += 1
        self.person_info["me"]["total"] += 1

        if type_ == "1": # recv
            self.person_info["me"]["num_recv"] += 1
            self.person_info["me"]["msgs"].append(['r', date, body])

            self.person_info[person]["num_recv"] += 1
            self.person_info[person]["msgs"].append(['r', date, body])

        elif type_ == "2": # sent
            self.person_info["me"]["num_sent"] += 1
            self.person_info["me"]["msgs"].append(['s', date, body])

            self.person_info[person]["num_sent"] += 1
            self.person_info[person]["msgs"].append(['s', date, body])

    def sort_person_most_used_words(self, person):
        msgs = self.person_info[person]["msgs"]
        recv_word_count = {}
        sent_word_count = {}

        # Count words
        for recv_or_send, _, msg in msgs:
            words = msg.split(" ")
            for word in words:
                if recv_or_send == 's':
                    if word in sent_word_count:
                        sent_word_count[word] += 1
                    else:
                        sent_word_count[word] = 1

                elif recv_or_send == 'r':
                    if word in recv_word_count:
                        recv_word_count[word] += 1
                    else:
                        recv_word_count[word] = 1

        self.person_info[person]["recv_word_count_sorted"] = self.sort_dict_by_value(recv_word_count)
        self.person_info[person]["sent_word_count_sorted"] = self.sort_dict_by_value(sent_word_count)

    def sort_msg_dates(self, person):
        msgs = self.person_info[person]["msgs"]
        recv_msg_dates = [date for recv_or_sent, date, __ in msgs if recv_or_sent == 'r']
        sent_msg_dates = [date for recv_or_sent, date, __ in msgs if recv_or_sent == 's']
        msg_dates = [[recv_or_sent, date] for recv_or_sent, date, __ in msgs]

        self.person_info[person]["recv_dates_sorted"] = sorted(recv_msg_dates)
        self.person_info[person]["sent_dates_sorted"] = sorted(sent_msg_dates)
        self.person_info[person]["dates_sorted"] = sorted(msg_dates, key=itemgetter(1))

    def get_avg_response_times(self, person):
        dates = self.person_info[person]["dates_sorted"]

        my_count = 0
        their_count = 0
        my_response_sum = 0
        their_response_sum = 0

        for i, msg in enumerate(dates):
            if i + 1 == len(dates):
                break

            sent_or_recv, date = msg
            next_sent_or_recv, next_date = dates[i + 1]

            minutes_diff = date_funcs.get_minutes_diff(date, next_date)

            # Ignore very long response times to try and exclude overnight responses
            diff_cutoff = 60 * 6
            if minutes_diff < diff_cutoff:
                if sent_or_recv == 'r' and next_sent_or_recv == 's':
                    my_count += 1
                    my_response_sum += minutes_diff

                elif sent_or_recv == 's' and next_sent_or_recv == 'r':
                    their_count += 1
                    their_response_sum += minutes_diff

                else:
                    pass

        try:
            self.person_info[person]["avg_response_time_me"] = round(my_response_sum / my_count, 2)
        except:
            self.person_info[person]["avg_response_time_me"] = "N/A"
        try:
            self.person_info[person]["avg_response_time_them"] = round(their_response_sum / their_count, 2)
        except:
            self.person_info[person]["avg_response_time_them"] = "N/A"

    def sort_dict_by_value(self, dict):
        return list(reversed(sorted(dict.items(), key=lambda kv: kv[1])))

    def get_messages_per_day(self, person):
        try:
            # Build range of dates
            dates_sorted = self.person_info[person]["dates_sorted"]
            first_date = dates_sorted[0][1]
            last_date = dates_sorted[-1][1]

            first_date = date_funcs.unix_date_to_datetime(first_date)
            last_date = date_funcs.unix_date_to_datetime(last_date)

            date_range = last_date - first_date

            dates_to_plot = [date_funcs.date_to_Y_m_d(first_date + timedelta(i)) for i in range(date_range.days + 2)]

            self.person_info[person]["date_range"] = dates_to_plot

            # Build sent and receive count for each date
            recv_date_count = [0] * len(dates_to_plot)
            sent_date_count = [0] * len(dates_to_plot)

            try:
                recv_dates = self.person_info[person]["recv_dates_sorted"]
                sent_dates = self.person_info[person]["sent_dates_sorted"]
                for date in recv_dates:
                    date = date_funcs.date_to_Y_m_d(date, is_unix=True)
                    range_idx = dates_to_plot.index(date)
                    recv_date_count[range_idx] += 1

                for date in sent_dates:
                    date = date_funcs.date_to_Y_m_d(date, is_unix=True)
                    range_idx = dates_to_plot.index(date)
                    sent_date_count[range_idx] += 1

                self.person_info[person]["recv_msgs_by_day"] = recv_date_count
                self.person_info[person]["sent_msgs_by_day"] = sent_date_count
            except:
                print("index error...")

        except:
            pass

    def get_most_common_words_str(self, sorted_words, words_to_return = 5):
        most_used_words = []
        words_found = 0
        for word, count in sorted_words:
            if words_found == words_to_return:
                break

            if word not in self.ignored_words:
                most_used_words.append([word, count])
                words_found += 1

        return ' | '.join(str(word) + ", " + str(count) for word, count in most_used_words)


import tensorflow as tf
import tensorflow.keras.layers as layers
import numpy as np

from src.models.gan import utils

class OldReducedGenerator(tf.keras.layers.Layer):

    def __init__(self, depthlen=288, gen_features=8, numparticle=6,
                 overscale=10.0, expscale=False,
                 activation=tf.keras.activations.tanh, **kwargs):
        super().__init__(**kwargs)

        self.depthlen = depthlen
        self.gen_features = gen_features

        self.overscale = 10.0
        self.expscale = expscale

        self.labelmerger = utils.LabelMerger(numparticle=numparticle,
                                             expscale=expscale)

        self.activation = activation
        if self.expscale:
            last_activation = tf.keras.activations.elu
        else:
            last_activation = tf.keras.activations.sigmoid

        self._nheight = 4
        self._nwidth = int(np.ceil(self.depthlen / (2 * 2 * 2 * 2 * 2)))
        self._nfilter = 32

        # dense label/noise combination
        self.layer_combine1 = layers.Dense(512, activation=self.activation)
        self.layer_combine2 = layers.Dense(self._nheight*self._nwidth*self._nfilter,
                                           activation=self.activation)

        # 1st transpose then conv
        self.layer_tc1 = layers.Conv2DTranspose(64, (2,5), strides=(1,2),
                                                padding="same",
                                                activation=self.activation)
        self.layer_rc1 = layers.Conv2D(64, (2,5), padding="same",
                                       activation=self.activation)

        # 2nd transpose then conv
        self.layer_tc2 = layers.Conv2DTranspose(32, (2,10), strides=(1,2),
                                                padding="same",
                                                activation=self.activation)
        self.layer_rc2 = layers.Conv2D(32, (2,10), padding="same",
                                       activation=self.activation)

        # 3rd transpose then conv
        self.layer_tc3 = layers.Conv2DTranspose(16, (2,10), strides=(1,2),
                                                padding="same",
                                                activation=self.activation)
        self.layer_rc3 = layers.Conv2D(16, (2,10), padding="same",
                                       activation=self.activation)

        # 4th transpose then conv
        self.layer_tc4 = layers.Conv2DTranspose(32, (2,5), strides=(1,2),
                                                padding="same",
                                                activation=self.activation)
        self.layer_rc4 = layers.Conv2D(32, (2,5), padding="same",
                                       activation=self.activation)

        # 5th transpose then conv
        self.layer_tc5 = layers.Conv2DTranspose(16, (2,5), strides=(1,2),
                                                padding="same",
                                                activation=self.activation)
        self.layer_rc5 = layers.Conv2D(self.gen_features, (self._nheight, 5),
                                       padding="valid",
                                       activation=last_activation)

    @tf.function
    def call(self, inputs, training=False):
        label = inputs[0]
        noise = inputs[1][0]

        labeltensor = self.labelmerger(label)
        tensor = tf.concat([labeltensor, noise], -1)

        # dense
        tensor = self.layer_combine1(tensor)
        tensor = self.layer_combine2(tensor)

        # reshape for convs
        tensor = tf.reshape(tensor, [-1, self._nheight, self._nwidth, self._nfilter])

        # convs
        tensor = self.layer_tc1(tensor)
        tensor = self.layer_rc1(tensor)

        tensor = self.layer_tc2(tensor)
        tensor = self.layer_rc2(tensor)

        tensor = self.layer_tc3(tensor)
        tensor = self.layer_rc3(tensor)

        tensor = self.layer_tc4(tensor)
        tensor = self.layer_rc4(tensor)

        tensor = self.layer_tc5(tensor)
        tensor = tf.pad(tensor, ([0,0], [0,0], [2,2], [0,0]))
        tensor = self.layer_rc5(tensor)

        # remove dimensions of size 1
        # shape should be [None, self.depthlen, self.gen_features]
        tensor = tf.squeeze(tensor, [1])[:,0:self.depthlen,:]

        if self.expscale:
            tensor = tf.math.exp(-1.0*(tensor + 1.0 - tf.math.log(self.overscale)))
        else:
            tensor = tensor * self.overscale

        return tensor


#sum of values
from collections import OrderedDict
d = {}
n = int(input())
for i in range(n):
    a = input().split(' ')
    key = ' '.join(a[:-1])
    if key in d:
        d[key] += int(a[-1])
    else:
        d[key] = int(a[-1])

d = OrderedDict(sorted(d.items(), key=lambda x: x[0]))
for i in d.items():
    print(i[0], i[1])


#!/usr/bin/env python3

from __future__ import print_function

import pythonpath; pythonpath
from x19290.myglobalid import my_global_id

import anchor; anchor
from tw.secretparser import secrets

from tweepy import OAuthHandler, API as Twitter, Cursor

from argparse import ArgumentParser
from itertools import count

_me = my_global_id()[0]
def main():
    args = _ArgumentParser().parse_args()
    type_, id_, = args.type, args.id
    if type_ == r'user':
        if not id_:
            id_ = _me
    else:
        if not id_:
            raise AssertionError(r'missing argument')
    try:
        n = int(args.limit)
        limit = range(n)
    except ValueError:
        limit = count()
    g = secrets(_me)
    access_token, access_token_secret, consumer_key, consumer_secret = g
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = Twitter(auth)
    tw=api.statuses_lookup(r'634782508345860096')
    i = None
    try:
        for i, tw in zip(limit, Cursor(api.user_timeline, id=id_).items()):
            print(tw.id, tw.text, sep='----\n', end='\n\n')
    finally:
        print(0 if i is None else i + 1)

class _ArgumentParser(ArgumentParser):
    def __init__(self):
        super(_ArgumentParser, self).__init__()
        self.add_argument(r'-t', r'--type', default=r'user')
        self.add_argument(r'-i', r'--id')
        self.add_argument(r'-l', r'--limit', default=3)

if __name__ == r'__main__':
    main()


import os
import re
import shutil
import unittest

from ia_markov import MarkovModel
from ia_markov import POSMarkov

class TestMarkov(unittest.TestCase):

    def test_model(self):
        path = 'tmp'
        os.makedirs(path)
        os.chdir(path)
        m = MarkovModel()
        m.train_model('FuturistManifesto')
        assert isinstance(m.model.make_sentence(), str)
        p = POSMarkov()
        # p.train_model('FuturistManifesto')
        # assert isinstance(p.model.make_sentence(), str)
        os.chdir(os.pardir)
        shutil.rmtree('tmp', ignore_errors=True)

    def test_clean_sentences(self):
        # test single regex pattern
        regex = re.compile(r'[0-9]*^.*?\s[A-Z]*\s[A-Z]*')
        m = MarkovModel(exclude=regex)
        sentences = ['138 TEST STRING', 'second string is here']
        self.assertEquals(len(sentences), 2)
        cleaned = m._clean_sentences(sentences)
        self.assertEquals(len(cleaned), 1)

        # test list of regex patterns
        reg_list = [regex]
        regex2 = re.compile(r'[a-z]*\s[a-z]*\s[a-z]*\s[a-z]')
        reg_list.append(regex2)
        m = MarkovModel(exclude=reg_list)
        cleaned = m._clean_sentences(sentences)
        self.assertEquals(len(cleaned), 0)

if __name__ == '__main__':
    unittest.main()


import sys, os
from argparse import ArgumentParser

# Options
parser = ArgumentParser(description ='Script to run the training and evaluate it')
parser.add_argument("--multi", action='store_true', default=False, help="Binary or categorical crossentropy")
#parser.add_argument("-g", "--gpu", default=1, help="Run on gpu's (Need to be in deepjetLinux3_gpu env)", const=1)
parser.add_argument("-i", help="Training dataCollection.dc", default=None, metavar="FILE")
parser.add_argument("-t", help="Testing dataCollection.dc", default=None, metavar="FILE")
parser.add_argument("-d",  help="Training output dir", default=None, metavar="PATH")
parser.add_argument("-o",  help="Eval output dir", default=None, metavar="PATH")
parser.add_argument("--batch",  help="Batch size, default = 2000", default=2000, metavar="INT")
parser.add_argument("--epochs",  help="Epochs, default = 50", default=50, metavar="INT")
opts=parser.parse_args()

sampleDatasets_pf_cpf_sv = ["db","pf","cpf","sv"]
sampleDatasets_cpf_sv = ["db","cpf","sv"]
sampleDatasets_sv = ["db","sv"]

#select model and eval functions
from models.DeepJet_models_final import conv_model_final as trainingModel
from DeepJetCore.training.training_base import training_base
from funcs import loadModel, evaluate

inputDataset = sampleDatasets_pf_cpf_sv
trainDir = opts.d
inputTrainDataCollection = opts.t
inputTestDataCollection = opts.i
LoadModel = True
removedVars = None

if True:
    evalModel = loadModel(trainDir,inputTrainDataCollection,trainingModel,LoadModel,inputDataset,removedVars)
    evalDir = opts.o

    from DeepJetCore.DataCollection import DataCollection
    testd=DataCollection()
    testd.readFromFile(inputTestDataCollection)

    if os.path.isdir(evalDir):
        raise Exception('output directory: %s must not exists yet' %evalDir)
    else:
        os.mkdir(evalDir)

    df = evaluate(testd, evalModel, evalDir)


# -*- coding: utf-8 -*-
from django.contrib.auth.decorators import login_required
from django.contrib.auth import authenticate, login, logout
from django.template import RequestContext
from django.template.loader import render_to_string
from django.shortcuts import render_to_response, HttpResponseRedirect, redirect
from django.views.decorators.csrf import csrf_protect
from django.db.models import Max
from django.template.defaultfilters import slugify
from models import *
from forms import *
from django.views.decorators.vary import vary_on_cookie

@vary_on_cookie
def categoria (request,link):
    sidebar = Sidebar.objects.all().order_by('orden')
    try:
        categoria = Categoria_historial.objects.get(link=link).ultima_version()
        articulos = Articulo_historial.objects.filter(articulos2__categorias = categoria.historial_link)
        return render_to_response('categorias/categoria.html', {'categoria':categoria,'articulos':articulos,'link':link,'sidebar':sidebar },RequestContext(request))
    except:
        mensaje = 'Esta categoria no existe.'
        crear_link = '/crear/categoria/?link=%s' %link
        return render_to_response('no_encontrado.html', {'mensaje':mensaje,'crear_link':crear_link,'sidebar':sidebar},RequestContext(request))

@vary_on_cookie
def categoria_comentarios (request,link):
    echo = ''
    sidebar = Sidebar.objects.all().order_by('orden')
    categoria = Categoria_historial.objects.get(link=link)
    if request.method == 'POST':
        if request.user.is_authenticated():
            form = Comentario_usuario_form(request.POST)
        else:
            form = Comentario_form(request.POST)
        if form.is_valid():
            if request.user.is_authenticated():
                comentario = form.save(commit=False)
                comentario.autor = request.user
                comentario.save()
                categoria.comentarios.add(comentario)
            else:
                comentario = form.save()
                categoria.comentarios.add(comentario)
            echo = 'Mensaje enviado'
    else:
        if request.user.is_authenticated():
            form = Comentario_usuario_form()
        else:
            form = Comentario_form()
    return render_to_response('categorias/categoria_comentarios.html', {'categoria':categoria,'link':link,'echo':echo,'form':form,'sidebar':sidebar},RequestContext(request))

@login_required
def categoria_instancia (request,id):
    sidebar = Sidebar.objects.all().order_by('orden')
    categoria = Categoria.objects.get(id=id)
    return render_to_response('categorias/categoria_instancia.html', {'categoria':categoria,'sidebar':sidebar },RequestContext(request))

@vary_on_cookie
def categoria_recientes (request,link):
    sidebar = Sidebar.objects.all().order_by('orden')
    categoria = Categoria_historial.objects.get(link=link).ultima_version()
    articulos = Articulo_historial.objects.filter(articulos2__categorias = categoria.historial_link).order_by('-articulos2__fecha').distinct()
    return render_to_response('categorias/categoria_recientes.html', {'categoria':categoria,'articulos':articulos,'link':link,'sidebar':sidebar },RequestContext(request))

@login_required
def categoria_historial(request,link):
    sidebar = Sidebar.objects.all().order_by('orden')
    categorias = Categoria.objects.filter(historial_link__link=link).order_by('-fecha')
    return render_to_response('categorias/categoria_historial.html', {'categorias':categorias,'link':link,'sidebar':sidebar},RequestContext(request))

@login_required
@csrf_protect
def editar_categoria (request,link=''):
    sidebar = Sidebar.objects.all().order_by('orden')
    if request.method == 'POST':
        form = Categoria_form(request.POST, request.FILES)
        if form.is_valid():
            if link:
                historial_link = Categoria_historial.objects.get(link=link)
                form = Categoria_form(request.POST, request.FILES, instance=historial_link.ultima_version())
                categoria = form.save(commit=False)
                categoria.historial_link = historial_link
                categoria.ultimo_autor = request.user
                categoria.id = None
                categoria.save()
                form.save_m2m()
            else:
                categoria = form.save(commit=False)
                if request.POST.get('link',False):
                    link = request.POST['link']
                else:
                    link = slugify(categoria.nombre)
                historial_link = Categoria_historial.objects.create(link=link)
                categoria.historial_link = historial_link
                categoria.ultimo_autor = request.user
                categoria.save()
                form.save_m2m()
            return redirect('/categoria/%s/' %link)
    else:
        if link:
            form = Categoria_form(instance=Categoria_historial.objects.get(link=link).ultima_version())
        else:
            if request.GET.get('id',False): ## carga objeto del historial
                form  = Categoria_form(instance=Categoria.objects.get(id=request.GET.get('id')))
            else: ## carga objeto vacio
                form = Categoria_form()
    return render_to_response('form_cat_art_img.html', {'form':form,'link':link,'tipo':'categoria','sidebar':sidebar },RequestContext(request))


from django.shortcuts import render
from django.http import HttpResponse, response
from .models import Order, Product,Contact,OrderUpdate
from math import ceil
import json
def index(request):
    products= Product.objects.all()
    allProds=[]
    catprods= Product.objects.values('category', 'id')
    cats= {item["category"] for item in catprods}
    for cat in cats:
        prod=Product.objects.filter(category=cat)
        n = len(prod)
        nSlides = n // 4 + ceil((n / 4) - (n // 4))
        allProds.append([prod, range(1,nSlides), nSlides])

    params={'allProds':allProds }
    return render(request,"shop/index.html", params)

def about(request):
    return render(request,'shop/about.html')

def contact(request):
    if request.method=='POST':
        name=request.POST.get('name','')
        email=request.POST.get('email','')
        phone=request.POST.get('phone','')
        desc=request.POST.get('desc','')

        contact=Contact(name=name,email=email,phone=phone,desc=desc)
        contact.save()
        success=True
        return render(request,'shop/contact.html',{'success':success})
    return render(request,'shop/contact.html')

def search(request):
    return render(request,'shop/search.html')

def productView(request,myid):
    #fetch the product using id
    product=Product.objects.filter(id=myid)

    return render(request,'shop/prodview.html',{'product':product[0]})

def checkout(request):
    if request.method=='POST':
        itemsJson=request.POST.get('itemsJson','')
        name=request.POST.get('name','')
        email=request.POST.get('email','')
        phone=request.POST.get('phone','')
        locality=request.POST.get('locality','')
        city=request.POST.get('city','')
        state=request.POST.get('state','')
        zip=request.POST.get('zip','')
        address=request.POST.get('address','')
        order=Order(name=name,email=email,address=address,locality=locality,city=city,state=state,zip=zip,phone=phone,item_json=itemsJson)
        order.save()
        order_update=OrderUpdate(order_id=order.order_id,update_desc="The order has been placed")
        order_update.save()
        thank=True
        id=order.order_id
        return render(request,'shop/checkout.html',{'thank':thank,'id':id})
    return render(request,'shop/checkout.html')

def tracker(request):
    if request.method=='POST':
        orderId=request.POST.get('orderId','')
        email=request.POST.get('email','')
        # return HttpResponse(f"{orderId} and {email}")
        try:
            order=Order.objects.filter(order_id=orderId,email=email)
            if len(order)>0:
                update=OrderUpdate.objects.filter(order_id=orderId)
                updates=[]
                for item in update:
                    updates.append({'text':item.update_desc,'time':item.timestamp})
                    response=json.dumps([updates,order[0].item_json],default=str)
                return HttpResponse(response)
            else:
                return HttpResponse("{}")

        except Exception as e:
            return HttpResponse('{}')

    return render(request,'shop/tracker.html')


import pytz

from django.core.management.base import BaseCommand
from faker import Faker
from faker.providers import date_time, lorem
from tqdm import tqdm

from events.models import Event

class Command(BaseCommand):
    help = "Creates test events"

    def handle(self, *args, **kwargs):
        Event.objects.all().delete()

        fake = Faker()
        fake.add_provider(date_time)
        fake.add_provider(lorem)
        print("Creating 50 fake events")
        for _ in tqdm(range(50)):
            Event.objects.create(
                title=fake.sentence(nb_words=6),
                description=fake.text(max_nb_chars=500),
                start_datetime=fake.date_time_this_month(
                    before_now=True,
                    after_now=False,
                    tzinfo=pytz.timezone("America/Chicago"),
                ),
                end_datetime=fake.date_time_this_month(
                    before_now=False,
                    after_now=True,
                    tzinfo=pytz.timezone("America/Chicago"),
                ),
                event_image="https://via.placeholder.com/300",
            )


"""One of the main goals of this lesson is to help you develop your ability to look at some code and identify its time complexity—and then describe this time complexity using Big O notation.

We will use this graph as a referece and reminder of the importance of the run time of an algorithm. We have the number of inputs (n) on the X axis and the the number of operations required (N) on the Y axis.

Drawing
<all-comparison-computational-complexity.svg>
"Comparison of computational complexity" by Cmglee. Used under CC BY-SA 4.0.
"""
"""Quadratic Example"""
# O(n^2)

def Quad_Example(our_list):
    for first_loop_item in our_list:
        for second_loop_item in our_list:
            print("Items: {}, {}".format(first_loop_item, second_loop_item))

Quad_Example([1, 2, 3, 4])
"""
Items: 1, 1
Items: 1, 2
Items: 1, 3
Items: 1, 4
Items: 2, 1
Items: 2, 2
Items: 2, 3
Items: 2, 4
Items: 3, 1
Items: 3, 2
Items: 3, 3
Items: 3, 4
Items: 4, 1
Items: 4, 2
Items: 4, 3
Items: 4, 4
CPU times: user 3 µs, sys: 0 ns, total: 3 µs
Wall time: 6.2 µs
"""

"""Log Linear Example"""
# O(nlogn)
# Don't worry about how this algorithm works, we will cover it later in the course!

def Log_Linear_Example(our_list):
    if len(our_list) < 2:
        return our_list

    else:
        mid = len(our_list)//2
        left = our_list[:mid]
        right = our_list[mid:]
        Log_Linear_Example(left)
        Log_Linear_Example(right)
        i = 0
        j = 0
        k = 0
        while i < len(left) and j < len(right):
            if left[i] < right[j]:
                our_list[k] = left[i]
                i += 1
            else:
                our_list[k] = right[j]
                j += 1
            k += 1
        while i < len(left):
            our_list[k] = left[i]
            i += 1
            k += 1
        while j < len(right):
            our_list[k] = right[j]
            j += 1
            k += 1
        return our_list

Log_Linear_Example([56, 23, 11, 90, 65, 4, 35, 65, 84, 12, 4, 0])

"""Linear Example"""
# O(n)

def Linear_Example(our_list):
    for item in our_list:
        print("Item: {}".format(item))

Linear_Example([1, 2, 3, 4])

"""Logarithmic Example"""
# O(logn)

def Logarithmic_Example(number):
    if number == 0:
        return 0

    elif number == 1:
        return 1

    else:
        return Logarithmic_Example(number-1)+Logarithmic_Example(number-2)

Logarithmic_Example(29)
"""Constant Example"""
# O(1)

def Constant_Example(our_list):
    return our_list.pop()

Constant_Example([1, 2, 3, 4])

"""
What is the run time analysis of the following code: -> O(1)
"""

def main(x, y):
    if True:
        z = x + y
    for i in range(10):
        z += i
    return z

"""
What is the run time analysis of the following code: -> O(n^2)
"""

def main(list_1, list_2):

    count = 0

    for item_1 in list_1:
        for item_2 in list_2:
            if item_1 == item_2:
                count += 1

    return count

"""
What is the simplification of this run time analysis: 4n^2 + 3n + 7 ? -> n^2
"""

# https://www.bigocheatsheet.com/
# https://wiki.python.org/moin/TimeComplexity

Course/Data structures and algorithms/1.Introduction/1.6.Efficiency/7.EfficiencyPractice.py


import math
import sys
sys.setrecursionlimit(10 ** 9)

class MyLib:
    get_line: iter = lambda: map(int, sys.stdin.readline().rstrip().split())
    get_input: int = lambda: int(input())
    print_list = lambda lst: print('\n'.join(map(str, lst)))
    find_indexes = lambda x, lst: [idx for idx, value in enumerate(lst) if value == x]
    # a, b, c = get_line()
    # array:list = list(get_line())
    # a = get_input()
    # print_list(array)
    # for index in find_indexes(value, array)

def print_postorder(start, end, inputs: list):
    if start > end:
        return
    else:
        current = inputs[start]
        mid = end + 1

        # mid 찾기
        # 현재 노드 ~ end
        for index in range(start + 1, end + 1):
            if current < inputs[index]:
                mid = index
                break

        # start+1 부터 mid-1 은 왼쪽 트리
        # mid 부터 end는 오른쪽 트리
        print_postorder(start + 1, mid - 1, inputs)
        print_postorder(mid, end, inputs)
        print(current)

def main():
    # 이진검색트리를 전위순회(root->left->right)한 결과를 보고 후위순회(left->right->root)로 바꾸기.
    inputs = []

    for line in sys.stdin:
        for var in line.split():
            inputs.append(int(var))
    print_postorder(0, len(inputs) - 1, inputs)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

# @Time    : 2018/11/23 13:59
# @Author  : jian
# @File    : class_model_nn.py
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.naive_bayes import MultinomialNB, GaussianNB, BernoulliNB

data = pd.read_csv("data/data.csv")
# 构建特征 目标值

X = data.iloc[0: -1, 1:7]
Y = data.iloc[1:, 7:]

# print(X.shape, Y.shape)
# minmax = MinMaxScaler()
# X = minmax.fit_transform(X)
# #测试集 训练集划分
x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.25)
# print(x_test)
# 建模
nn = MultinomialNB()

nn.fit(x_train, y_train)

y_predict = nn.predict(x_test)

pre = nn.predict([[1, 10, 25, 27, 30, 32]])
# pre = nn.predict([[3, 5, 11, 15, 20, 23]])
# pre = nn.predict([[1, 3, 6, 10, 11, 29]])
print(pre, "pre")
# dict = nn.predict(x_test)
# print(nn.out_activation_)

# 预测准确率
print(nn.score(x_test, y_test))

t = np.arange(len(y_test))
plt.figure()
plt.plot(t, y_test, 'r-', linewidth=2, label='Test_b')
plt.plot(t, y_predict, 'g-', linewidth=2, label='Predict_b')
plt.xticks(tuple(x for x in range(len(y_test))))
plt.grid()
plt.show()


from PIL import Image, ImageDraw
from tkinter.filedialog import asksaveasfile
from tkinter.filedialog import askopenfilename
import os
import cv2
import random
import time
import numpy as np
import statistics as stat

def file_open_general():
    f = askopenfilename(title="Select file", filetypes=(("jpg files", "*.jpg"), ("png files", "*.png"),
                                                        ("jpeg files", "*.jpeg")))
    if f:
        return Image.open(f)

def file_open_cv():
    f = askopenfilename(title="Select file", filetypes=(("jpg files", "*.jpg"), ("png files", "*.png"),
                                                        ("jpeg files", "*.jpeg")))
    if f:
        return cv2.imread(f, cv2.IMREAD_COLOR)

def file_save(im):
    f = asksaveasfile(mode='w', defaultextension=".png")
    if f:  # asksaveasfile return `None` if dialog closed with "cancel".
        abs_path = os.path.abspath(f.name)
        im.save(abs_path)
    f.close()

def gray_scale_filter_pil():
    image = file_open_general()
    draw = ImageDraw.Draw(image)
    width = image.size[0]
    height = image.size[1]
    pix = image.load()
    for i in range(width):
        for j in range(height):
            r = pix[i, j][0]
            g = pix[i, j][1]
            b = pix[i, j][2]
            s = round(0.2126 * r + 0.7152 * g + 0.0722 * b)
            draw.point((i, j), (s, s, s))
    image.show()
    file_save(image)
    return image
def noise1(image):
    wigth = image.size[0]
    heigth = image.size[1]
    image1 = Image.new("RGB", (wigth, heigth))
    pix = image1.load()
    pix1 = image.load()
    for x in range(image1.size[0]):
        for y in range(image1.size[1]):
            rand = random.randint(-180, 180)
            rand = rand//2
            ImageDraw.Draw(image1).point((x, y), (rand, rand, rand))
    image1.show()
    for x in range(image.size[0]):
        for y in range(image.size[1]):
            r1 = pix[x, y][0]
            g1 = pix[x, y][1]
            b1 = pix[x, y][2]
            r = pix1[x, y][0]
            g = pix1[x, y][1]
            b = pix1[x, y][2]
            r += r1
            g += g1
            b += b1
            if (r < 0):
                r = 0
            if (g < 0):
                g = 0
            if (b < 0):
                b = 0
            if (r > 255):
                r = 255
            if (g > 255):
                g = 255
            if (b > 255):
                b = 255
            ImageDraw.Draw(image).point((x, y), (r, g, b))
    image.show()
    file_save(image)
    return image

def center_point(image):
    image = image.convert('HSV')
    for x in range(1, image.size[0]-1):
        for y in range(1, image.size[1]-1):
            hsv = image.load()
            H = hsv[x, y][0]
            S = hsv[x, y][1]
            V = hsv[x, y][2]
            min1 = 1000
            max1 = 0
            for i in range(x-1,x+1):
                for j in range(y-1,y+1):
                        s1 = hsv[i, j][2]
                        if (s1 > max1):
                            max1 = s1
                        if (s1 < min1):
                            min1 = s1
            V = (max1 + min1) // 2
            ImageDraw.Draw(image).point((x, y), (H, S, V))
    image = image.convert('RGB')
    image.show()

def noise2(image):
    image = image.convert('HSV')
    pix = image.load()
    #pix1 = image.load()
    for x in range(image.size[0]):
        for y in range(image.size[1]):
            rand = random.randint(5, 80)
            if rand == 80:
                V = random.randint(1, 100)
            else:
                V = pix[x, y][2]
            ImageDraw.Draw(image).point((x, y), (pix[x,y][0], pix[x,y][1], V))
    image = image.convert('RGB')
    image.show()
    file_save(image)
    return image

def brightHSV(img1, value):
    imgHSV = cv2.cvtColor(img1, cv2.COLOR_BGR2HSV)
    h, s, v = cv2.split(imgHSV)
    v += value
    img = cv2.cvtColor(imgHSV, cv2.COLOR_HSV2BGR)
    return img

def harmonic(image):
    image = image.convert('HSV')
    wigth = image.size[0]
    heigth = image.size[1]
    image1 = Image.new("HSV", (wigth, heigth))
    hsv = image.load()
    for x in range(2, image.size[0]-1):
        for y in range(2, image.size[1]-1):
            H = hsv[x, y][0]
            S = hsv[x, y][1]
            V = hsv[x, y][2]
            sum0 = 0
            sum1 = 0
            sum11 = 0
            for i in range(x-2, x+2):
                for j in range(y-2, y+2):
                    if hsv[i, j][0] != 0:
                        sum0 += 1/hsv[i, j][0]
                    if hsv[i, j][1] != 0:
                        sum1 += 1/hsv[i, j][1]
                    if hsv[i, j][2] != 0:
                        sum11 += 1/hsv[i, j][2]
            sum0 = sum0 / 9
            sum1 = sum1 / 9
            sum11 = sum11 / 9
            if sum0 != 0:
                H = (round)(1/sum0)
            if sum1 != 0:
                S = (round)(1/sum1)
            if sum11 != 0:
                V = (round)(1/sum11)
            ImageDraw.Draw(image1).point((x, y), (H, S, V))
    image1 = image1.convert('BGR')
    img = brightHSV(image1, 10)
    image = Image.fromarray(img)
    image1.show()
    file_save(image1)
    return image1

def opencv_grsc_denoise():
    cv2.startWindowThread()
    image = file_open_cv()
    #blank_image = np.zeros((image.shape), np.uint8)
    result = cv2.fastNlMeansDenoising(image, None, 10, 7, 21)
    cv2.namedWindow('Press esc to close')
    cv2.imshow('Press esc to close', result)
    gray_image = Image.fromarray(result)
    file_save(gray_image)
    # cv2.waitKey()
    cv2.destroyAllWindows()

print("Choose preferred mode:\n"
      "1 - Noise addition filter + Denoising using Center point filter\n"
      "2 - Denoising using Center point filter\n"
      "3 - Denoising using Harmonic mean filter\n"
      "4 - Denoising using fastNlMeansDenoising filter(OpenCV) for grayscale images\n")
print('Input mode № = ')
mode = input()
if mode == '1':
     img = Image.open('C:/Users/Kate/Pictures/filters/image6.png')
     d = noise1(img)
elif mode == '2':
     img_noise = Image.open('C:/Users/Kate/Pictures/filter_results/image_noised_.png')
     d = center_point(img_noise)
elif mode == '3':
    img_noise = Image.open('C:/Users/Kate/Pictures/filter_results/image_noised_.png')
    d = harmonic(img_noise)
elif mode == '4':
    opencv_grsc_denoise()

#img_noise = Image.open('C:/Users/Kate/Pictures/index.jpeg')
#d = center_point(img_noise)
"""def opencv_denoise():
        cv2.startWindowThread()
        image = file_open_cv()
        # blank_image = np.zeros((image.shape), np.uint8)
        result = cv2.fastNlMeansDenoisingColored(image, None, 20, 10, 21, 7)
        cv2.namedWindow('Press esc to close')
        cv2.imshow('Press esc to close', result)
        gray_image = Image.fromarray(result)
        file_save(gray_image)
        # cv2.waitKey()
        cv2.destroyAllWindows()"""

#img_noise = noise2(Image.open('C:/Users/Kate/Pictures/filters/image6.png'))


def _dataflow_analyze(self):
    self._build_graph()
    live_in = defaultdict(set)
    worklist = list(range((len(self._ops) - 1), (- 1), (- 1)))
    while worklist:
        i = worklist.pop(0)
        live_in[i] = set(self._live_in[i])
        for s in self._successors[i]:
            self._live_out[i] |= self._live_in[s]
        self._live_in[i] = (self._uses[i] | (self._live_out[i] - self._defs[i]))
        if (live_in[i] != self._live_in[i]):
            for d in self._presuccessors[i]:
                worklist.append(d)


from subprocess import Popen, STDOUT, PIPE
from bob.errors import BobError
from bob import git

import shutil
def create_virtualenv(name):
    job = Popen(['virtualenv', '/tmp/{}'.format(name)], stderr=STDOUT, stdout=PIPE)
    job.wait()
    if job.returncode != 0:
        raise BobError("Cannot create virtualenv.")
    output, _ = job.communicate()
    return output.decode('utf-8')

def remove_virtualenv(name):
    try:
        shutil.rmtree('/tmp/{}'.format(name))
    except OSError:
        raise BobError("Cannot remove virtualenv")


#!/usr/bin/env python3.5
# -*- coding:utf-8 -*

import os
import platform
import urllib
import requests
import json
import logging

# -----------------------------------------------------------------------------
# v1.1 - For changes visit https://github.com/trini148/Code-Snippets/WallpaperFetcherBing.py
# Works on Ubuntu Gnome, Unity and XFCE.
# -----------------------------------------------------------------------------

LOGGING_ENABLED = False
LOGGING_PATH = os.path.join('/home', 'elias', 'Documents', 'Startup Scripts')
IMAGE_PATH = os.path.join('/home', 'elias', 'Pictures', 'BingDailyWallpaper')
LOCALE = 'de-DE'

# -----------------------------------------------------------------------------
# Do not modify unless you know what you're doing!
# -----------------------------------------------------------------------------

class Logger:
    logging_enabled = False

    def __init__(self, logging_enabled, logging_path):
        Logger.logging_enabled = logging_enabled

        if Logger.logging_enabled:
            # Setting up logger
            self.log_handler = logging.FileHandler(os.path.join(logging_path, 'Log.log'))
            self.log_formatter = logging.Formatter(
                '%(asctime)s - %(message)s', datefmt='%d/%m/%Y %I:%M:%S %p')
            self.log_handler.setFormatter(self.log_formatter)

            self.logger = logging.getLogger('LOG')
            self.logger.addHandler(self.log_handler)
            self.logger.setLevel(logging.DEBUG)
            self.logger.info('Starting daily wallpaper fetcher ...')

    def info(self, message):
        if Logger.logging_enabled:
            self.logger.info(message)

class FileSystem:
    @staticmethod
    def create_image_path(image_path):
        if not os.path.exists(image_path):
            os.makedirs(image_path)

    @staticmethod
    def create_logging_path(logging_path):
        if not os.path.exists(logging_path):
            os.makedirs(logging_path)

class Wallpaper:
    def __init__(self, logging_enabled, logging_path, image_path, locale):
        self.logger = Logger(logging_enabled=logging_enabled,
                             logging_path=logging_path)
        self.json_object = json.loads(
            requests.post('%s%s' % ('http://www.bing.com/HPImageArchive.aspx?format=js&idx=0&n=1&mkt=', locale),
                          json={"key": "value"}).text)['images'][0]
        self.image_name = '%s' % self.json_object['enddate']
        self.image_url = '%s%s' % ("http://www.bing.de",
                                   self.json_object['url'])
        self.image_path_local = os.path.join(image_path, '%s%s' % (self.image_name, '.jpg'))

    def __is_downloaded(self):
        return os.path.isfile(self.image_path_local)

    def download_today(self):
        if not self.__is_downloaded():
            self.logger.info("Download link for today's image (%s) is '%s' ..." % (self.image_name, self.image_url))
            self.logger.info("Fetching image from web ...")
            urllib.request.urlretrieve(self.image_url, self.image_path_local)
            self.logger.info("Fetching image from web ... Done!")

    def set_today(self):
        self.logger.info("Setting the wallpaper ...")
        if platform.system().startswith("Lin"):
            if os.environ.get("XDG_CURRENT_DESKTOP") == "XFCE":  # XFCE
                os.system(
                    "xfconf-query --channel xfce4-desktop --property /backdrop/screen0/monitor0/workspace0/last-image --set %s"
                    % self.image_path_local)
                os.system(
                    "xfconf-query --channel xfce4-desktop --property /backdrop/screen0/monitor1/workspace0/last-image --set %s"
                    % self.image_path_local)
            else:  # Unity & Gnome
                os.system("gsettings set org.gnome.desktop.background picture-uri file://'%s'" % self.image_path_local)

        self.logger.info("Setting the wallpaper ... Done!")

FileSystem.create_image_path(image_path=IMAGE_PATH)
FileSystem.create_logging_path(logging_path=LOGGING_PATH)

wallpaper = Wallpaper(logging_enabled=LOGGING_ENABLED,
                      logging_path=LOGGING_PATH,
                      image_path=IMAGE_PATH,
                      locale=LOCALE)
wallpaper.download_today()
wallpaper.set_today()


import cv2
import numpy as np
import os
import math

n = 0
mean1 = 190
mean2 = 150
mean3 = 250

std1 = 10
std2 = 10
std3 = 10

path = "Dataset1/Green/Training/"

def calculateGaussian(x, mean, std):
    gauss = (1 / (std * math.sqrt(2 * math.pi))) * (math.exp(-((x - mean) ** 2) / (2 * std ** 2)))
    return gauss

def get_images():
    images = []
    for image in os.listdir(path):
        images.append(image)
    return images

def em_gmm(pixel):
    global n, mean1, mean2, mean3, std1, std2, std3

    while n != 40:
        prob1 = []
        prob2 = []
        prob3 = []

        b1 = []
        b2 = []
        b3 = []

        for pix in pixel:
            p1 = calculateGaussian(pix, mean1, std1)
            prob1.append(p1)
            p2 = calculateGaussian(pix, mean2, std2)
            prob2.append(p2)
            p3 = calculateGaussian(pix, mean3, std3)
            prob3.append(p3)
            b1.append((p1 * (1 / 3.0)) / (p1 * (1 / 3.0) + p2 * (1 / 3.0) + p3 * (1 / 3.0)))
            b2.append((p2 * (1 / 3.0)) / (p1 * (1 / 3.0) + p2 * (1 / 3.0) + p3 * (1 / 3.0)))
            b3.append((p3 * (1 / 3.0)) / (p1 * (1 / 3.0) + p2 * (1 / 3.0) + p3 * (1 / 3.0)))

        mean1 = np.sum(np.array(b1) * np.array(pixel)) / np.sum(np.array(b1))
        mean2 = np.sum(np.array(b2) * np.array(pixel)) / np.sum(np.array(b2))
        mean3 = np.sum(np.array(b3) * np.array(pixel)) / np.sum(np.array(b3))

        std1 = (np.sum(np.array(b1) * ((np.array(pixel)) - mean1) ** 2) / np.sum(np.array(b1))) ** (1 / 2.0)
        std2 = (np.sum(np.array(b2) * ((np.array(pixel)) - mean2) ** 2) / np.sum(np.array(b2))) ** (1 / 2.0)
        std3 = (np.sum(np.array(b3) * ((np.array(pixel)) - mean3) ** 2) / np.sum(np.array(b3))) ** (1 / 2.0)
        print("Iteration = ", n)
        print(mean1, mean2, mean3)
        print(std1, std2, std3)
        n = n + 1
    return mean1, mean2, mean3, std1, std2, std3

def append_all_pixel(images):
    pixel = []
    for image in images:
        image = cv2.imread("%s%s" % (path, image))
        image = image[:, :, 1]
        r, c = image.shape
        for j in range(0, r):
            for m in range(0, c):
                pix = image[j][m]
                pixel.append(pix)
    return pixel

def main():
    images = get_images()
    pixel = append_all_pixel(images)
    mean1, mean2, mean3, std1, std2, std3 = em_gmm(pixel)
    print("\n")
    print('final mean- ', mean1, mean2, mean3)
    print('final std- ', std1, std2, std3)

if __name__ == '__main__':
    main()

Buoy detection using GMM/Code/GMM_Green.py


import numpy as np

# 二次元配列 (行列)
a = np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]])
print(a,"\n\n",
      a.flags,"\n",
      a.ndim,"## 次元数\n", ## 次元数
      a.size,"## 要素数\n", ## 要素数
      a.shape, "## 各次元の要素数 (行数, 列数)\n",## 各次元の要素数 (行数, 列数)
      a.itemsize,"## 1要素のバイト数\n", ## 1要素のバイト数
      a.strides,"## 24バイトで次の行、8バイトで次の列\n", ## 24バイトで次の行、8バイトで次の列
      a.nbytes, "## 配列全体のバイト数 (size*itemsize)\n", ## 配列全体のバイト数 (size*itemsize)
      a.dtype,"## 要素のデータ型(後述)")  ## 要素のデータ型(後述)


import collections
import time
from heapq import heappush, heapify, heappop, heappushpop

class ScheduledItem:
    """ A class that describes a scheduled callback.

    This class is never created by the user; instead, pygame creates and
    returns an instance of this class when scheduling a callback.

    If you hold on to instance of this class, do not modify any values of it.
    """
    __slots__ = ['func', 'interval', 'last_ts', 'next_ts']

    def __init__(self, func, last_ts, next_ts, interval):
        self.func = func
        self.interval = interval
        self.last_ts = last_ts
        self.next_ts = next_ts

    def __lt__(self, other):
        try:
            return self.next_ts < other.next_ts
        except AttributeError:
            return self.next_ts < other

class Scheduler:
    """Class for scheduling functions.
    """

    def __init__(self, time_function=time.perf_counter):
        """Initialise a Clock, with optional custom time function.

        :Parameters:
            `time_function` : function
                Function to return the elapsed time of the application,
                in time units.
        """
        super().__init__()
        self.cumulative_time = 0.0
        self._time = time_function
        self._last_ts = -1
        self._times = collections.deque(maxlen=10)
        self._scheduled_items = list()
        self._next_tick_items = list()

    def _get_nearest_ts(self):
        """Schedule from now, unless now is sufficiently close to last_ts, in
        which case use last_ts.  This clusters together scheduled items that
        probably want to be scheduled together.  The old (pre 1.1.1)
        behaviour was to always use self.last_ts, and not look at ts.  The
        new behaviour is needed because clock ticks can now be quite
        irregular, and span several time units.
        """
        last_ts = self._last_ts
        ts = self._time()
        if ts - last_ts > 0.2:
            last_ts = ts
        return last_ts

    def _get_soft_next_ts(self, last_ts, interval):
        def taken(ts, e):
            """Return True if the given time has already got an item
            scheduled nearby.
            """
            for item in self._scheduled_items:
                if item.next_ts is None:
                    continue
                elif abs(item.next_ts - ts) <= e:
                    return True
                elif item.next_ts > ts + e:
                    return False
            return False

        # Binary division over interval:
        #
        # 0                          interval
        # |--------------------------|
        # 5  3   6   2   7  4  8   1          Order of search
        #
        # i.e., first scheduled at interval,
        # then at            interval/2
        #       then at            interval/4
        #       then at            interval*3/4
        #       then at            ...
        #
        # Schedule is hopefully then evenly distributed for any interval,
        # and any number of scheduled functions.

        next_ts = last_ts + interval
        if not taken(next_ts, interval / 4):
            return next_ts

        dt = interval
        divs = 1
        while True:
            next_ts = last_ts
            for i in range(divs - 1):
                next_ts += dt
                if not taken(next_ts, dt / 4.):
                    return next_ts
            dt /= 2
            divs *= 2

            # Avoid infinite loop in pathological case
            if divs > 16:
                return next_ts

    def schedule(self, func, delay=0.0, repeat=False, soft=False):
        """
        Schedule a function to be run sometime in the future

        The function should have a prototype that includes ``dt`` as the
        first argument, which gives the elapsed time, in time units, since the
        last clock tick.

            def callback(dt):
                pass

        Limitations
        ===========

        There is a hard limit of 10 items that can be scheduled on
        every tick.  This limit reduces the power and performance
        impact of the clock on mobile and battery operated computers.

        This 10 maximum only effects callbacks scheduled with
        repeat=True and delay=0.0.

        A runtime error will be raised if the maximum is reached.

        Unscheduling
        ============

        If callback returns False (not None), then it will not be
        scheduled again.

        Soft Scheduling
        ===============

        This is useful for functions that need to be called regularly,
        but not relative to the initial start time.
        for example: events which need to occur regularly -- if all audio
        updates are scheduled at the same time
        (for example, mixing several tracks of a music score, or playing
        multiple videos back simultaneously), the resulting load on the
        CPU is excessive for those intervals but idle outside.  Using
        the soft interval scheduling, the load is more evenly distributed.

        :param func: Function to be called
        :param delay: Delay in time unit until it is called
        :param repeat: Function will be repeated every 'delay' units
        :param soft: See notes about Soft Scheduling
        :rtype: ScheduledItem
        :return: Reference to scheduled item
        """
        last_ts = self._get_nearest_ts()
        if soft:
            assert (delay > 0.0)
            next_ts = self._get_soft_next_ts(last_ts, delay)
            last_ts = next_ts - delay
        next_ts = last_ts + delay

        interval = delay if repeat else 0.0

        item = ScheduledItem(func, last_ts, next_ts, interval)
        if next_ts == 0.0:
            self._next_tick_items.append(item)
            if len(self._next_tick_items) > 10:
                raise RuntimeError
        else:
            heappush(self._scheduled_items, item)
        return item

    def tick(self):
        """Cause clock to update self and call scheduled functions.

        This updates the clock's internal measure of time and returns
        the difference since the last update (or since the clock was created).

        Will call any scheduled functions that have elapsed.

        :rtype: float
        :return: The number of time units since the last "tick", or 0 if this
        was
                 the first tick.
        """
        delta_t = self.set_time(self._time())
        self._times.append(delta_t)
        self.call_scheduled_functions(delta_t)
        return delta_t

    def get_interval(self):
        """Get the average amount of time passed between each tick.

        Useful for calculating FPS if this clock is used with the display.
        Returned value is averaged from last 10 ticks.

        Value will be zero if before 1st tick.

        :rtype: float
        :return: The number of time units since the last "tick", or 0.0 if this
                 was the first tick.
        """
        try:
            return sum(self._times) / len(self._times)
        except ZeroDivisionError:
            return 0.0

    def set_time(self, time_stamp):
        """Set the clock manually and do not call scheduled functions.  Return
        the difference in time from the last time clock was updated.

        :Parameters:
            `time_stamp` : float
                This will become the new value of the clock.  Setting the clock
                to negative values will have undefined results.

        :rtype: float
        :return: The number of time units since the last update, or 0.0 if this
                 was the first update.

        """
        if self._last_ts < 0:
            delta_t = 0.0
            self._last_ts = time_stamp
        else:
            delta_t = time_stamp - self._last_ts
        self.cumulative_time += delta_t
        self._last_ts = time_stamp
        return delta_t

    def call_scheduled_functions(self, dt):
        """Call scheduled functions that elapsed on the last `update_time`.

        :Parameters:
            dt : float
                The elapsed time since the last update to pass to each
                scheduled function.

        :rtype: bool
        :return: True if any functions were called, otherwise False.
        """
        scheduled_items = self._scheduled_items
        now = self._last_ts
        result = False

        # handle items scheduled for each tick
        if self._next_tick_items:
            result = True
            for item in list(self._next_tick_items):
                retval = item.func(dt)
                # do not change the following line to "if not retval"!
                # some items will return None, but False is a special value
                if retval == False:
                    self._next_tick_items.remove(item)

        # check the next scheduled item that is not called each tick
        # if it is scheduled in the future, then exit
        try:
            item = scheduled_items[0]
            if item.next_ts > now:
                return result
        except IndexError:
            return result

        # we have at least one item that is due to be called
        result = True
        get_soft_next_ts = self._get_soft_next_ts

        # wherever this value is true the current item will be pushed
        # into the heap.  it essentially means that the current
        # scheduled item is important and needs stay scheduled.
        replace = False

        while scheduled_items:

            # get the next item to be called
            head = scheduled_items[0]
            if head.next_ts > now:
                if replace:
                    heappush(scheduled_items, item)
                replace = False
                break

            # the scheduler will hold onto a reference to an item in
            # case it needs to be rescheduled.  it is more efficient
            # to push and pop the heap in one call than to make two
            # calls.
            if replace:
                item = heappushpop(scheduled_items, item)
            else:
                item = heappop(scheduled_items)

            # call the function associated with the scheduled item
            retval = item.func(now - item.last_ts)

            if item.interval:
                # callbacks can unschedule themselves by returning false
                replace = not retval == False
                item.next_ts = item.last_ts + item.interval
                item.last_ts = now

                # the execution time of this item has already passed
                # so it must be rescheduled
                if item.next_ts <= now:
                    if now - item.next_ts < 0.05:
                        item.next_ts = now + item.interval
                    else:
                        # missed by significant amount, do a soft reschedule
                        # to avoid lumping everything together
                        # in this case, the next dt will not be accurate
                        item.next_ts = get_soft_next_ts(now, item.interval)
                        item.last_ts = item.next_ts - item.interval
            else:
                # replace is False, so this item will not be rescheduled
                replace = False

        # it is possible that the loop exited while an important item
        # was waiting to be pushed into the heap.
        if replace:
            heappush(scheduled_items, item)

        return result

    def get_idle_time(self):
        """Get the time until the next item is scheduled.

        :rtype: float
        :return: Time until the next scheduled event in time units, or ``None``
                 if there is no event scheduled.
        """
        if self._next_tick_items:
            return 0.0

        try:
            next_ts = self._scheduled_items[0].next_ts
            return max(next_ts - self._time(), 0.)
        except IndexError:
            return None

    def unschedule(self, func):
        """Remove a function from the schedule.

        If the function appears in the schedule more than once, all occurrences
        are removed.  If the function was not scheduled, no error is raised.

        :Parameters:
            `func` : function
                The function to remove from the schedule.

        """

        def remove(list_):
            resort = False
            remove_ = list_.remove
            for i in list(i for i in list_ if i.func is func):
                remove_(i)
                resort = True
            return resort

        remove(self._next_tick_items)
        if remove(self._scheduled_items):
            # this will restructure the heap
            heapify(self._scheduled_items)

class Clock(Scheduler):
    """Schedules stuff like a Scheduler, and includes time limiting functions

    WIP
    """


"""A module for the Amino Acid Insertion Classifier."""
from typing import List
from .set_based_classifier import SetBasedClassifier
from variation.schemas.classification_response_schema import ClassificationType

class AminoAcidInsertionClassifier(SetBasedClassifier):
    """The Amino Acid Insertion Classifier class."""

    def classification_type(self) -> ClassificationType:
        """Return the Amino Acid Insertion classification type."""
        return ClassificationType.AMINO_ACID_INSERTION

    def exact_match_candidates(self) -> List[List[str]]:
        """Return the exact match token type candidates."""
        return [
            ['AminoAcidInsertion'],
            ['GeneSymbol', 'AminoAcidSubstitution', 'AminoAcidInsertion'],
            ['AminoAcidInsertion', 'GeneSymbol'],
            ['GeneSymbol', 'AminoAcidInsertion'],
            ['HGVS', 'AminoAcidInsertion'],
            ['ReferenceSequence', 'AminoAcidInsertion'],
            ['LocusReferenceGenomic', 'AminoAcidInsertion']
        ]


# Copyright 2018 HTCondor Team, Computer Sciences Department,
# University of Wisconsin-Madison, WI.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Tuple, Iterable, Dict, Optional, Callable, Iterator, Any
import logging

import time
import shutil
from pathlib import Path
import itertools

import htcondor

from . import htio, exceptions, maps, options, settings

logger = logging.getLogger(__name__)

def maps_dir_path() -> Path:
    """The path to the directory where map directories are stored."""
    return settings['HTMAP_DIR'] / settings['MAPS_DIR_NAME']

def map_dir_path(map_id: str) -> Path:
    """The path to the directory for the given ``map_id``."""
    return maps_dir_path() / map_id

def get_schedd():
    """Get the :class:`htcondor.Schedd` that represents the HTCondor scheduler."""
    s = settings.get('HTCONDOR.SCHEDD', default = None)
    if s is not None:
        return htcondor.Schedd(s)

    return htcondor.Schedd()

def map(
    map_id: str,
    func: Callable,
    args: Iterable[Any],
    map_options: options.MapOptions = None,
    **kwargs,
) -> maps.Map:
    """
    Map a function call over a one-dimensional iterable of arguments.
    The function must take a single positional argument and any number of keyword arguments.

    The same keyword arguments are passed to *each call*, not mapped over.

    Parameters
    ----------
    map_id
        The ``map_id`` to assign to this map.
    func
        The function to map the arguments over.
    args
        An iterable of arguments to pass to the mapped function.
    kwargs
        Any additional keyword arguments are passed as keyword arguments to the mapped function.
    map_options
        An instance of :class:`htmap.MapOptions`.

    Returns
    -------
    result :
        A :class:`htmap.Map` representing the map.
    """
    args = ((arg,) for arg in args)
    args_and_kwargs = zip(args, itertools.repeat(kwargs))
    return create_map(
        map_id,
        func,
        args_and_kwargs,
        map_options = map_options,
    )

def starmap(
    map_id: str,
    func: Callable,
    args: Optional[Iterable[tuple]] = None,
    kwargs: Optional[Iterable[Dict[str, Any]]] = None,
    map_options: options.MapOptions = None,
) -> maps.Map:
    """
    Map a function call over aligned iterables of arguments and keyword arguments.
    Each element of ``args`` and ``kwargs`` is unpacked into the signature of the function,
    so their elements should be tuples and dictionaries corresponding to position and keyword arguments of the mapped function.

    Parameters
    ----------
    map_id
        The ``map_id`` to assign to this map.
    func
        The function to map the arguments over.
    args
        An iterable of tuples of positional arguments to unpack into the mapped function.
    kwargs
        An iterable of dictionaries of keyword arguments to unpack into the mapped function.
    map_options
        An instance of :class:`htmap.MapOptions`.

    Returns
    -------
    result :
        A :class:`htmap.Map` representing the map.
    """
    if args is None:
        args = ()
    if kwargs is None:
        kwargs = ()

    args_and_kwargs = zip_args_and_kwargs(args, kwargs)
    return create_map(
        map_id,
        func,
        args_and_kwargs,
        map_options = map_options,
    )

id_gen = itertools.count()

def get_transient_map_id() -> str:
    return f'tmp-{int(time.time())}-{next(id_gen)}'

class TransientMap:
    def __init__(self, map: maps.Map):
        self._map = map

    def __repr__(self):
        return f'<{self.__class__.__name__}(map = {self._map})>'

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._cleanup()

        return False  # re-raise exceptions

    def __iter__(self):
        try:
            yield from self._map
        finally:
            self._cleanup()

    def __del__(self):
        self._cleanup()

    def _cleanup(self):
        try:
            self._map.remove()
            logger.debug(f'removed transient map {self._map.map_id}')
        except exceptions.MapWasRemoved:
            pass

def transient_map(
    func: Callable,
    args: Iterable[Any],
    map_options: options.MapOptions = None,
    **kwargs,
) -> TransientMap:
    """
    As :func:`htmap.map`, except that it doesn't need a ``map_id``, it returns an iterator over the outputs, and the map is immediately removed after use.
    """
    m = map(
        map_id = get_transient_map_id(),
        func = func,
        args = args,
        map_options = map_options,
        **kwargs,
    )

    return TransientMap(m)

def transient_starmap(
    func: Callable,
    args: Optional[Iterable[tuple]] = None,
    kwargs: Optional[Iterable[Dict[str, Any]]] = None,
    map_options: options.MapOptions = None,
) -> TransientMap:
    """
    As :func:`htmap.starmap`, except that it doesn't need a ``map_id``, it returns an iterator over the outputs, and the map is immediately removed after use.
    """
    m = starmap(
        map_id = get_transient_map_id(),
        func = func,
        args = args,
        kwargs = kwargs,
        map_options = map_options,
    )

    return TransientMap(m)

class MapBuilder:
    def __init__(
        self,
        map_id: str,
        func: Callable,
        map_options: options.MapOptions = None,
    ):
        self.func = func
        self.map_id = map_id
        self.map_options = map_options

        self.args = []
        self.kwargs = []

        self._result = None

        logger.debug(f'initialized map builder for map {map_id} for {self.func}')

    def __repr__(self):
        return f'<{self.__class__.__name__}(func = {self.func}, map_options = {self.map_options})>'

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        # if an exception is raised in the with, re-raise without submitting jobs
        if exc_type is not None:
            logger.exception(f'map builder for map {self.map_id} aborted due to')
            return False

        self._result = starmap(
            map_id = self.map_id,
            func = self.func,
            args = self.args,
            kwargs = self.kwargs,
            map_options = self.map_options
        )

        logger.debug(f'finished executing map builder for map {self.map_id}')

    def __call__(self, *args, **kwargs):
        """Adds the given inputs to the map."""
        self.args.append(args)
        self.kwargs.append(kwargs)

    @property
    def result(self) -> maps.Map:
        """
        The :class:`Map` associated with this :class:`MapBuilder`.
        Will raise :class:`htmap.exceptions.NoResultYet` when accessed until the ``with`` block for this :class:`MapBuilder` completes.
        """
        if self._result is None:
            raise exceptions.NoResultYet('result does not exist until after with block')
        return self._result

    def __len__(self):
        """The length of a :class:`MapBuilder` is the number of inputs it has been sent."""
        return len(self.args)

def build_map(
    map_id: str,
    func: Callable,
    map_options: options.MapOptions = None,
) -> MapBuilder:
    """
    Return a :class:`MapBuilder` for the given function.

    Parameters
    ----------
    map_id
        The ``map_id`` to assign to this map.
    func
        The function to map over.
    map_options
        An instance of :class:`htmap.MapOptions`.

    Returns
    -------
    map_builder :
        A :class:`MapBuilder` for the given function.
    """
    return MapBuilder(
        map_id = map_id,
        func = func,
        map_options = map_options,
    )

def build_transient_map(
    map_id: str,
    func: Callable,
    map_options: options.MapOptions = None,
) -> TransientMap:
    """
    As :func:`htmap.build_map`, except that it doesn't need a ``map_id``, it returns an iterator over the outputs, and the map is immediately removed after use.
    """
    mb = MapBuilder(
        map_id = map_id,
        func = func,
        map_options = map_options,
    )

    return TransientMap(mb.result)

def create_map(
    map_id: str,
    func: Callable,
    args_and_kwargs: Iterator[Tuple[Tuple, Dict]],
    map_options: Optional[options.MapOptions] = None,
) -> maps.Map:
    """
    All map calls lead here.
    This function performs various checks on the ``map_id``,
    constructs a submit object that represents the map for HTCondor,
    saves all of the map's definitional data to the map directory,
    and submits the map job,
    returning the map's :class:`Map`.

    Parameters
    ----------
    map_id
        The ``map_id`` to assign to this map.
    func
        The function to map the arguments over.
    args_and_kwargs
        The arguments and keyword arguments to map over - the output of :func:`zip_args_and_kwargs`.
    map_options
        An instance of :class:`htmap.MapOptions`.

    Returns
    -------
    result :
        A :class:`htmap.Map` representing the map.
    """
    raise_if_map_id_is_invalid(map_id)
    raise_if_map_id_already_exists(map_id)

    logger.debug(f'creating map {map_id}...')

    map_dir = map_dir_path(map_id)
    try:
        make_map_dir_and_subdirs(map_dir)
        htio.save_func(map_dir, func)
        hashes = htio.save_args_and_kwargs(map_dir, args_and_kwargs)
        htio.save_hashes(map_dir, hashes)

        submit_obj, itemdata = options.create_submit_object_and_itemdata(
            map_id,
            map_dir,
            hashes,
            map_options,
        )
        htio.save_submit(map_dir, submit_obj)
        htio.save_itemdata(map_dir, itemdata)

        logger.debug(f'submitting map {map_id}...')
        cluster_id = execute_submit(
            submit_object = submit_obj,
            itemdata = itemdata,
        )

        logger.debug(f'map {map_id} was assigned clusterid {cluster_id}')

        with (map_dir / 'cluster_ids').open(mode = 'a') as file:
            file.write(str(cluster_id) + '\n')

        with (map_dir / 'cluster_ids').open() as file:
            cluster_ids = [int(cid.strip()) for cid in file]

        r = maps.Map(
            map_id = map_id,
            cluster_ids = cluster_ids,
            submit = submit_obj,
            hashes = hashes,
        )

        logger.info(f'submitted map {map_id}')

        return r
    except BaseException as e:
        # something went wrong during submission, and the job is malformed
        # so delete the entire map directory
        # the condor bindings should prevent any jobs from being submitted
        logger.exception(f'map submission for map {map_id} aborted due to')
        shutil.rmtree(str(map_dir.absolute()))
        logger.debug(f'removed malformed map directory {map_dir}')
        raise e

def raise_if_map_id_already_exists(map_id: str):
    """Raise a :class:`htmap.exceptions.MapIdAlreadyExists` if the ``map_id`` already exists."""
    if map_dir_path(map_id).exists():
        raise exceptions.MapIdAlreadyExists(f'the requested map_id {map_id} already exists (recover the Map, then either use or delete it).')

INVALID_FILENAME_CHARACTERS = {
    '/',
    '\\',  # backslash
    '<',
    '>',
    ':',
    '"',
    '|',
    '?',
    '*',
    '.',
    ' ',
}

def raise_if_map_id_is_invalid(map_id: str):
    """Raise a :class:`htmap.exceptions.InvalidMapId` if the ``map_id`` contains any invalid characters."""
    invalid_chars = set(map_id).intersection(INVALID_FILENAME_CHARACTERS)
    if len(invalid_chars) != 0:
        raise exceptions.InvalidMapId(f'These characters in map_id {map_id} are not valid: {invalid_chars}')

MAP_SUBDIR_NAMES = (
    'inputs',
    'outputs',
    'job_logs',
    'cluster_logs',
)

def make_map_dir_and_subdirs(map_dir):
    """Create the input, output, and log subdirectories inside the map directory."""
    for path in (map_dir / d for d in MAP_SUBDIR_NAMES):
        path.mkdir(parents = True, exist_ok = True)

    logger.debug(f'created map directory {map_dir} and subdirectories')

def execute_submit(submit_object, itemdata) -> int:
    """
    Execute a map via the scheduler defined by the settings.
    Return the HTCondor cluster ID of the map's jobs.
    """
    schedd = get_schedd()
    with schedd.transaction() as txn:
        submit_result = submit_object.queue_with_itemdata(
            txn,
            1,
            iter(itemdata),
        )

        return submit_result.cluster()

def zip_args_and_kwargs(
    args: Iterable[Tuple],
    kwargs: Iterable[Dict],
) -> Iterator[Tuple[Tuple, Dict]]:
    """
    Combine iterables of arguments and keyword arguments into a zipped,
    filled iterator of arguments and keyword arguments (i.e., tuples and dictionaries).

    .. caution ::

        This function will happily run forever when given infinite iterator inputs.
        Be careful!

    Parameters
    ----------
    args
        A list of tuples.
    kwargs
        A list of dictionaries.

    Returns
    -------
    """
    iterators = [iter(args), iter(kwargs)]
    fills = {0: (), 1: {}}
    num_active = 2
    while True:
        values = []
        for i, it in enumerate(iterators):
            try:
                value = next(it)
            except StopIteration:
                num_active -= 1
                if num_active == 0:
                    return
                iterators[i] = itertools.repeat(fills[i])
                value = fills[i]
            values.append(value)
        yield tuple(values)


# When doing oversampling or undersampling, validation and train data for cross validation needs to be prepared manually as oversampling should not be done on Validation sets
# https://kiwidamien.github.io/how-to-do-cross-validation-when-upsampling-data.html

# Split data into Train and Test by CustomerID. Each CustomID may have multiple records
CustomerIDs = pd.unique(CustomerRatings.CustomerID)
CustomerIDs_Train = random.sample(list(CustomerIDs),np.ceil(len(CustomerIDs)*0.8).astype(int))
CustomerIDs_Test = set(CustomerIDs).difference(set(CustomerIDs_Train))

CustomerRatings_Train = CustomerRatings[CustomerRatings.CustomerID.isin(CustomerIDs_Train)]
CustomerRatings_Test = CustomerRatings[CustomerRatings.CustomerID.isin(CustomerIDs_Test)]

# Split first 80% rows into Train and remaining 20% rows into Test
Train = Df.iloc[:int(Df.shape[0]*0.80)] #Extracting first 80%
Test = Df.iloc[int(Df.shape[0]*0.80):] # Exracting remaining