import json
import csv
import os.path
import pdb
import re
import configparser
import string
import sys
import warnings
warnings.filterwarnings('ignore')
import math
from tqdm import tqdm
from aser.client import ASERClient
from aser.extract.aser_extractor import SeedRuleASERExtractor, DiscourseASERExtractor
from nltk.corpus import stopwords
stopwords = stopwords.words('english')
punctuation_string = string.punctuation  # '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'

config = configparser.ConfigParser()
config.read("paths.cfg")

'''
Please refer to: https://hkust-knowcomp.github.io/ASER/html/tutorial/get-started.html
run:
aser-server -n_workers 1 -n_concurrent_back_socks 10 -port 8000 -port_out 8001 -corenlp_path "YOUR_DIRECTORY/stanford-corenlp-3.9.2" -base_corenlp_port 8000
'''

class BM25(object):
    def __init__(self, docs):
        self.D = len(docs)
        self.avgdl = sum([len(doc)+0.0 for doc in docs]) / self.D
        self.docs = docs
        self.f = []  # 列表的每一个元素是一个dict，dict存储着一个文档中每个词的出现次数
        self.df = {} # 存储每个词及出现了该词的文档数量
        self.idf = {} # 存储每个词的idf值
        self.k1 = 1.5
        self.b = 0.75
        self.init()

    def init(self):
        for doc in self.docs:
            tmp = {}
            for word in doc:
                tmp[word] = tmp.get(word, 0) + 1  # 存储每个文档中每个词的出现次数
            self.f.append(tmp)
            for k in tmp.keys():
                self.df[k] = self.df.get(k, 0) + 1
        for k, v in self.df.items():
            self.idf[k] = math.log(self.D-v+0.5)-math.log(v+0.5)

    def sim(self, doc, index):
        score = 0
        for word in doc:
            if word not in self.f[index]:
                continue
            d = len(self.docs[index])
            score += (self.idf[word]*self.f[index][word]*(self.k1+1)
                      / (self.f[index][word]+self.k1*(1-self.b+self.b*d
                                                      / self.avgdl)))
        return score

    def simall(self, doc):
        scores = []
        for index in range(self.D):
            score = self.sim(doc, index)
            scores.append(score)
        return scores

def punc(s):
    s = s.replace('_comma_', ',')
    tmp = re.sub(r'([a-zA-Z])([,.!:;?])', r'\1 \2', s)
    s = re.sub(r'([,.!])([a-zA-Z])', r'\1 \2', tmp)
    s = s.strip()
    return s

def read_from_raw():
    tf = csv.reader(open(config["paths"]["raw_tst"]))
    dir = os.path.dirname(config["paths"]["prop_tst"])
    if os.path.exists(dir) is False:
        os.makedirs(dir)

    pcid = 'hit:0_conv:0'
    conv = {}
    conv['emotion'] = 'guilty'
    conv['situation'] = punc("I felt guilty when I was driving home one night and a person tried to fly into my lane_comma_ and didn't see me. I honked and they swerved back into their lane_comma_ slammed on their brakes_comma_ and hit the water cones.")
    conv['context'] = []
    test_data = []
    for i, line in enumerate(tf):
        if i == 0:
            continue
        cid = line[0]
        if cid == pcid:
            pcid = cid
            conv['context'].append(punc(line[5]))
        else:
            conv['context'] = conv['context']
            test_data.append(conv)
            conv = {}
            conv['emotion'] = line[2]
            conv['situation'] = punc(line[3])
            conv['context'] = [punc(line[5])]
            pcid = cid

    vf = csv.reader(open(config["paths"]["raw_dev"]))
    pcid = 'hit:3_conv:6'
    conv = {}
    conv['emotion'] = 'terrified'
    conv['situation'] = punc(
        "Today_comma_as i was leaving for work in the morning_comma_i had a tire burst in the middle of a busy road. That scared the hell out of me!")
    conv['context'] = []
    valid_data = []
    for i, line in enumerate(vf):
        if i == 0:
            continue
        cid = line[0]
        if cid == pcid:
            pcid = cid
            conv['context'].append(punc(line[5]))
        else:
            conv['context'] = conv['context']
            valid_data.append(conv)
            conv = {}
            conv['emotion'] = line[2]
            conv['situation'] = punc(line[3])
            conv['context'] = [punc(line[5])]
            pcid = cid

    trf = csv.reader(open(config["paths"]["raw_trn"]))
    pcid = 'hit:0_conv:1'
    conv = {}
    conv['emotion'] = 'sentimental'
    conv['situation'] = punc(
        "I remember going to the fireworks with my best friend. There was a lot of people_comma_ but it only felt like us in the world.")
    conv['context'] = []
    train_data = []
    for i, line in enumerate(trf):
        if i == 0:
            continue
        cid = line[0]
        if cid == pcid:
            pcid = cid
            conv['context'].append(punc(line[5]))
        else:
            conv['context'] = conv['context']
            train_data.append(conv)
            conv = {}
            conv['emotion'] = line[2]
            conv['situation'] = punc(line[3])
            conv['context'] = [punc(line[5])]
            pcid = cid

    print("====================================")
    max = 0
    t_data = []
    for t in test_data:
        for i, u in enumerate(t['context']):
            if (i+1)%2==0:
                new_t = t.copy()
                new_t['context'] = t['context'][:i]
                if len(new_t['context'])>max:
                    max = len(new_t['context'])
                new_t['response'] = t['context'][i]
                if new_t not in t_data:
                    t_data.append(new_t)
    # t_data = list(set(t_data))
    print('test: ', len(t_data))  # 2541
    json.dump(t_data, open(config["paths"]["prop_tst"],'w'), indent=4)

    v_data = []
    for t in valid_data:
        for i, u in enumerate(t['context']):
            if (i+1)%2==0:
                new_t = t.copy()
                new_t['context'] = t['context'][:i]
                if len(new_t['context'])>max:
                    max = len(new_t['context'])
                new_t['response'] = t['context'][i]
                if new_t not in v_data:
                    v_data.append(new_t)
    # v_data = list(set(v_data))
    print('valid: ', len(v_data))  # 2762
    json.dump(v_data, open(config["paths"]["prop_dev"], 'w'), indent=4)

    tr_data = []
    for t in train_data:
        for i, u in enumerate(t['context']):
            if (i+1)%2==0:
                new_t = t.copy()
                new_t['context'] = t['context'][:i]
                if len(new_t['context'])>max:
                    max = len(new_t['context'])
                new_t['response'] = t['context'][i]
                if new_t not in tr_data:
                    tr_data.append(new_t)
    # tr_data = list(set(tr_data))
    print('train:', len(tr_data))
    json.dump(tr_data, open(config["paths"]["prop_trn"], 'w'), indent=4)

    print('test: ', len(t_data))  # 8398
    print('valid: ', len(v_data))
    print('train: ', len(tr_data))
    print(max)  # the maximize total utterance number of the context and response is 6

def parse_one_old(text, client):
    '''
    transfer a sentence into events
    :param text:
    :return:
    '''
    text = text.strip()
    parsed_e, parsed_r = client.extract_eventualities_and_relations(text)

    res = []
    if parsed_e is None:
        text_s = text.split(' . ')
        for t in text_s:
            parsed_e, parsed_r = client.extract_eventualities_and_relations(t)  # this function not works now...
            if parsed_e is not None:
                res += parsed_e
        if res != []:
            parsed_e = res
    events = []
    if parsed_e is None:
        events = [text]
    else:
        for s in parsed_e:
            for e in s:
                events.append(' '.join(e.words))
        if events == []:
            events = [text]
    return events

def parse_one(text, client, extractor):
    '''
    transfer a sentence into events
    :param text:
    :return:
    '''
    text = text.strip()
    # parsed_e = aser_extractor.extract_from_text(text, in_order=True, use_lemma=True)
    parsed_e = extractor.extract_eventualities_from_text(text, in_order=True, use_lemma=True)
    # res = []
    # if parsed_e == [[]]:
    #     # text_s = text.split(' . ')
    #     text_s = re.split('([,.?;!])', text)
    #     text_s.append("")
    #     text_s = ["".join(i) for i in zip(text_s[0::2], text_s[1::2])]  # 每个分句子，带着分隔符
    #     for t in text_s:
    #         parsed_e = aser_extractor.extract_from_text(t, in_order=True, use_lemma=True)
    #         if parsed_e is not None:
    #             res += parsed_e
    #     if res != []:
    #         parsed_e = res

    events = []
    if parsed_e is [[]]:
        events = [text]
    else:
        for s in parsed_e:
            for e in s:
                events.append(' '.join(e.words))
        if events == []:
            events = [text]
    return events

def get_dialogue_event():
    client = ASERClient(port=8000, port_out=8001)

    aser_extractor = DiscourseASERExtractor(
        corenlp_path=sys.argv[1], corenlp_port=9000
    )
    paths = [config["paths"]["prop_tst"], config["paths"]["prop_dev"], config["paths"]["prop_trn"]]
    for path in paths:
        data = json.load(open(path, 'r'))
        new_data = []
        for i, item in enumerate(tqdm(data, desc='test')):
            context = item['context']
            dialogue_events = []  # each item contains the events of each utterance
            for utterance in context:
                # for i in punctuation_string:   # remove punctuation
                #     utterance = utterance.replace(i, '')
                u_e = parse_one(utterance, client=client, extractor=aser_extractor)
                dialogue_events.append(u_e)
            response = item['response']
            u_r = parse_one(response, client=client, extractor=aser_extractor)
            dialogue_events.append(u_r)
            item['events'] = dialogue_events
            new_data.append(item)
        json.dump(new_data, open(path, 'w'), indent=4)

def simplify_dialogue_event():
    paths = [config["paths"]["prop_tst"], config["paths"]["prop_dev"], config["paths"]["prop_trn"]]
    for path in paths:
        data = json.load(open(path, 'r'))
        new_data = []
        for i, item in enumerate(data):
            dialogue_events = item['events']  # each item contains the events of each utterance
            dialogue_sim_events = []
            for u_events in dialogue_events:
                new_u_events = []
                for event in u_events:
                    new_event = []
                    for w in event.split():
                        if w not in stopwords:
                            new_event.append(w)
                    new_u_events.append(' '.join(new_event))
                dialogue_sim_events.append(new_u_events)
            item['sim_events'] = dialogue_sim_events
            new_data.append(item)
        json.dump(new_data, open(path, 'w'), indent=4)

def extract_event_pairs():
    test_data = json.load(open(config["paths"]["prop_tst"], 'r'))
    with open(config["paths"]["test_event_pairs"], 'w') as f:
        for item in test_data:
            last_event = ""
            dialogue_events = item["events"]
            for utt_events in dialogue_events:
                for event in utt_events:
                    if last_event == "":
                        last_event = event
                        continue
                    else:
                        f.write(last_event + '\t' + event + '\n')
                        last_event = event

    dev_data = json.load(open(config["paths"]["prop_dev"], 'r'))
    with open(config["paths"]["dev_event_pairs"], 'w') as f:
        for item in dev_data:
            last_event = ""
            dialogue_events = item["events"]
            for utt_events in dialogue_events:
                for event in utt_events:
                    if last_event == "":
                        last_event = event
                        continue
                    else:
                        f.write(last_event + '\t' + event + '\n')
                        last_event = event

    train_data = json.load(open(config["paths"]["prop_trn"], 'r'))
    with open(config["paths"]["train_event_pairs"], 'w') as f:
        for item in train_data:
            last_event = ""
            dialogue_events = item["events"]
            for utt_events in dialogue_events:
                for event in utt_events:
                    if last_event == "":
                        last_event = event
                        continue
                    else:
                        f.write(last_event + '\t' + event + '\n')
                        last_event = event

def retrieve_event_from_context():
    doc = []
    doc_te = []
    doc_r = []
    atomic_dev = open('../../../data/atomic/event_triples/dev_event_triples.txt', 'r').readlines()
    atomic_train = open('../../../data/atomic/event_triples/train_event_triples.txt', 'r').readlines()
    atomic_test = open('../../../data/atomic/event_triples/test_event_triples.txt', 'r').readlines()
    for split in [atomic_train, atomic_dev, atomic_test]:
        for line in split:
            he, r, te = line.split('\t')
            he_words = he.strip().split(' ')
            if '' in he_words:
                he_words.remove('')
            doc.append(he_words)
            doc_te.append(te)
            doc_r.append(r)
    s = BM25(doc)
    print('finish bm25')

    for path in [config["paths"]["prop_tst"], config["paths"]["prop_dev"], config["paths"]["prop_trn"]]:
        data = json.load(open(path, 'r'))
        new_data = []
        for i, item in tqdm(enumerate(data), total=len(data)):
            context = item["context"]
            context_words = []
            for c in context:
                for w in c.split(' '):
                    w = w.strip()
                    if w in punctuation_string or w in stopwords:
                        continue
                    else:
                        context_words.append(w)
            scores = s.simall(context_words)
            retrieve_he = ' '.join(doc[scores.index(max(scores))])
            retrieve_te = doc_te[scores.index(max(scores))].strip()
            retrieve_r = doc_r[scores.index(max(scores))].strip()
            retrieve_kg = [retrieve_he, retrieve_r, retrieve_te]
            item['retrieve_event'] = retrieve_kg
            new_data.append(item)
        json.dump(new_data, open(path, 'w'), indent=4)

if __name__ == '__main__':
    # Step 1: read raw csv files.
    read_from_raw()

    # Step 2: parse each utterance into events. (it will takes some time, around 30 minutes)
    get_dialogue_event()

    # Step 3: remove stopwords from events.
    simplify_dialogue_event()

    # Step 4: extract event pairs from dataset for inferring relations from BERT-based relation classifier.
    extract_event_pairs()

    # Step 5: retrieve event from atomic using bm25 (one baseline)
    retrieve_event_from_context()


import json
import re
from dataclasses import dataclass
from typing import Dict, Optional, List, Union
import copy
from utils.util import *

# Formats long-form descriptions into a single thing
def combine_list_desc(
    base_description: str, listed_descriptions: Union[None, str, List[str]]
):
    long_desc = ""
    if mandate_list(listed_descriptions):
        li = []
        for i in listed_descriptions:
            body = i.get('description', "ERROR - UNPARSED DATA CONSULT PDF")
            if "name" in i:
                body = f"{i['name']}: {body}"
            li.append(f"
{body}
")
        long_desc = f"

{''.join(li)}
"
    return base_description + long_desc

class Processor:
    def __init__(self) -> None:
        # Tracks our top level data
        self.raw_traits = {}
        self.raw_units = {}
        self.created_summons = {}

    def ingest_all_raw(self) -> None:
        """
        Ingest all raw data to our raw_traits and raw_units properties
        """
        for path in walk_files(foe_root):
            if "traits.json" in path:
                with open(path) as f:
                    for rt in json.load(f):
                        rt = recursive_downcase(rt)
                        self.raw_traits[rt["name"]] = rt
            else:
                with open(path) as f:
                    loaded = json.load(f)
                    loaded = recursive_downcase(loaded)
                    self.raw_units[loaded["name"]] = loaded

    def inherit_into(self, target: Dict, source: Dict) -> Dict:
        """
        Merges source into target. Neither input is edited.
        Values from target are prioritized, but in the case of arrays will appear last
        """
        target = copy.deepcopy(target)
        source = copy.deepcopy(source)

        def _sub_inherit_list(key):
            target[key] = mandate_list(source.get(key)) + mandate_list(target.get(key))

        def _sub_inherit_attr(key):
            if key in source and key not in target:
                target[key] = source[key]

        # Combine traits, actions, conditional abilities
        _sub_inherit_list("traits")
        _sub_inherit_list("setup_traits")
        _sub_inherit_list("actions")
        _sub_inherit_list("conditional_abilities")

        # Inherit stats
        _sub_inherit_attr("faction")
        _sub_inherit_attr("class")
        _sub_inherit_attr("vitality")
        _sub_inherit_attr("speed")
        _sub_inherit_attr("defense")
        _sub_inherit_attr("fray_damage")
        _sub_inherit_attr("damage_die")

        # Finally handle recursion
        for super_source in mandate_list(source.get("inherits")):
            super_source = self.raw_units[super_source]
            target = self.inherit_into(target, super_source)

        return target

    # Lookup a trait or substitute a default
    def get_trait(self, trait):
        if trait in self.raw_traits:
            return copy.deepcopy(self.raw_traits[trait])
        else:
            return {"name": trait, "description": "No description."}

    def process(self):
        """
        Creates files for all intaken raw units
        """
        # Begin building
        for _name, data in self.raw_units.items():
            # Skip templates
            if data.get("type") == "Template":
                continue

            self.process_foe(data)

    def process_foe(self, data):
        """
        Processes an individual actor
        """
        ap = ActorProcessor(self, data)
        ap.process_as_foe()
        return ap

    def process_summon(self, data):
        """
        Processes an individual actor
        """
        key = data.get("name")
        if key in self.created_summons:
            return self.created_summons[key]
        ap = ActorProcessor(self, data)
        ap.process_as_summon()
        self.created_summons[key] = ap
        return ap

class ActorProcessor:
    """
    Stateful processor for ingesting actors
    """

    def __init__(self, processor: Processor, data: Dict) -> None:
        self.parent = processor
        self.data = data
        self.name = data.get("name", "Unknown")
        self.id = random_id()
        self.system = {}
        self.item_ids = []
        self.type = None

        # These accumulate and can be edited before actually being turned to items
        self.actions = []
        self.traits = []

    def raw_inherit(self):
        """
        Handle inheritance on our raw data
        """
        for super_source in mandate_list(self.data.get("inherits")):
            super_source = self.parent.raw_units[super_source]
            self.data = self.parent.inherit_into(self.data, super_source)

    def hydrate_traits(self):
        """
        Fills out the trait data on all traits on this
        """
        # Populate traits
        traits = [*set(mandate_list(self.data.get("traits")))]
        for t in traits:
            if isinstance(t, str):
                t = self.parent.get_trait(t)
            self.traits.append(t)

    def process_setup_info(self):
        """
        Processes our metadata for foe setup
        """
        # Add setup traits, post process to combine listed descriptions into descriptions
        setup = []
        for t in mandate_list(self.data.get("setup_traits")):
            base = f"
{t['name']}

" + t.get("description", "")
            full = combine_list_desc(base, t.get("listed_items"))
            setup.append(f"
{full}
")
        self.system["setup"] = "\n\n".join(setup)

    def process_conditional_abilities(self):
        for cond_ability in mandate_list(self.data.get("conditional_abilities")):
            # Check special classes, ignore those that don't apply
            add_reqs = []
            if cond_ability.get("is_special_classes"):
                add_reqs.append(f"Must be a {cond_ability['is_special_classes']}")
            if cond_ability.get("is_not_special_classes"):
                add_reqs.append(
                    f"Must not be a {cond_ability['is_not_special_classes']}"
                )
            if cond_ability.get("uses_special_template"):
                add_reqs.append(f"Must be a {cond_ability['uses_special_template']}")

            # If remov traits specified, don't actually remove, but instead mark as explicitly chaptered
            """
            for removal_target in mandate_list(cond_ability.get("remove_traits")):
                # Find the trait with the same name
                corr_trait = ([x for x in self.traits if x["name"] == removal_target])[0]
                # Mark it
                if "chapter" in cond_ability:
                    corr_trait["remove_at_chapter"] = cond_ability["chapter"]

            # Same for actions
            for removal_target in mandate_list(cond_ability.get("remove_actions")):
                # Find the trait with the same name
                corr_action = [x for x in self.actions if x["name"] == removal_target][0]
                # Mark it
                if "chapter" in cond_ability:
                    corr_action["remove_at_chapter"] = cond_ability["chapter"]
            """

            # Add traits, mark similarly
            for add_trait in mandate_list(cond_ability.get("traits")):
                if isinstance(add_trait, str):
                    add_trait = self.parent.get_trait(add_trait)
                if "chapter" in cond_ability:
                    chpt = cond_ability["chapter"]
                    add_trait["chapter"] = chpt
                add_trait["special_requirements"] = add_reqs
                self.traits.append(add_trait)

            # Add actions, mark similarly
            for add_action in mandate_list(cond_ability.get("actions")):
                if "chapter" in cond_ability:
                    chpt = cond_ability["chapter"]
                    add_action["chapter"] = chpt
                add_action["special_requirements"] = add_reqs
                self.actions.append(add_action)

    def process_as_summon(self):
        self.type = "summon"
        self.system["is_object"] = self.data.get("is_object", False)
        self.system["tags"] = self.data.get("tags", [])
        self.system["summon_effects"] = self.data.get("summon_effects", "")
        self.system["summon_actions"] = self.data.get("summon_actions", "")
        self.finalize()

    def process_as_foe(self):
        self.type = "foe"
        self.raw_inherit()
        self.hydrate_traits()

        # Do initial copying
        self.system["chapter"] = self.data.get("chapter", 1)
        self.system["faction"] = self.data.get("faction", "Unknown")
        foe_class = {}
        self.system["class"] = foe_class
        foe_class["color"] = self.data.get("class", "Special")
        foe_class["vitality"] = self.data.get("vitality", 4)
        foe_class["speed"] = self.data.get("speed", 4)
        foe_class["defense"] = self.data.get("defense", 4)
        foe_class["fray_damage"] = self.data.get("fray_damage", 1)
        foe_class["damage_die"] = self.data.get("damage_die", 6)
        self.actions.extend(mandate_list(self.data.get("actions")))

        foe_class["armor"] = 0
        if "Guard" in [t["name"] for t in self.traits]:
            foe_class["armor"] = 2

        self.process_setup_info()
        self.process_conditional_abilities()

        # Finally, convert our actions & traits
        for action in self.actions:
            self.process_action(action)

        for trait in self.traits:
            self.process_trait(trait)

        self.finalize()

    def process_action(self, data):
        """
        Uses an ItemProcessor to process the given item as an action
        """
        ip = ItemProcessor(self, data)
        ip.process_as_ability()

    def process_trait(self, data):
        """
        Uses an ItemProcessor to process the given item as an trait
        """
        ip = ItemProcessor(self, data)
        ip.process_as_trait()

    def finalize(self):
        """
        Create the actual document model and write it to a file
        """
        actor = {
            "name": self.name,
            "type": self.type,
            "_id": self.id,
            "system": self.system,
            "img": "icons/svg/mystery-man.svg",
            "items": self.item_ids,
            "effects": [],
            "folder": None,
            "sort": 0,
            "ownership": {"default": 0},
            "flags": {},
            "_stats": {
                "systemId": "icon",
                "systemVersion": "0.0.0",
                "coreVersion": "11.306",
                "createdTime": 1690238921300,
                "modifiedTime": 1690238921300,
                "lastModifiedBy": "pTpVgni8yGIuH7km",
            },
            "_key": f"!actors!{self.id}",
        }

        # Dump to appropriate file
        filename = f"{self.name}_{self.id[:3]}.json".replace(" ", "_")
        with open(pack_root / "foes" / "_source" / filename, "w") as f:
            f.write(json.dumps(actor, indent=4))

class ItemProcessor:
    """
    Stateful process for ingesting items
    """

    def __init__(self, actor: ActorProcessor, data: Dict) -> None:
        self.parent = actor
        self.data = data
        self.name = data.get("name", "Unknown Action")
        self.id = random_id()
        self.type = None
        self.system = {}

    def action_to_choices(self, action_data):
        # Convert an action to a choice (or, choices)
        choices = []
        all_tags = action_data.get("tags", [])
        ranges = []
        other_tags = []
        for tag in all_tags:
            if any(sym in tag for sym in ["range", "blast", "line", "arc", "burst"]):
                ranges.append(tag)
            else:
                other_tags.append(tag)

        # Siphon effects. Do it via entries rather than via keys so we preserve order
        effects = []
        for k, v in action_data.items():
            if k == "hit":
                effects.append(f"Hit: {v}")
            if k == "auto_hit":
                effects.append(f"Auto Hit: {v}")
            if k == "miss":
                effects.append(f"Miss: {v}")
            if k == "area_effect":
                effects.append(f"Area: {v}")
            if k == "effects":
                sub = mandate_list(v)
                effects.extend([f"Effect: {e}" for e in sub])
            if k == "description":
                effects.append(f"Effect: {v}")
            if k == "exceed":
                effects.append(f"Exceed: {v}")
            if k == "charge":
                effects.append(f"Charge: {v}")

        # Subprocess interrupts
        for interrupt in mandate_list(action_data.get("interrupts")):
            if interrupt.get("name", action_data["name"]) == action_data["name"]:
                interrupt["name"] += f"{action_data['name']} (interrupt)"  # Add a unique suffix if necessary
            choices = choices + self.action_to_choices(interrupt)

        # Subprocess combos
        if action_data.get("combo"):
            combo = action_data["combo"]
            combo["name"] += f"[COMBO]"  # Add a unique prefix
            combo["chapter"] = action_data.get("chapter", 1)  # Pass down chapter
            choices = choices + self.action_to_choices(action_data["combo"])

        # Subprocess summons
        summons = []
        for summon in mandate_list(action_data.get("summons")):
            summon["name"] += " " + CHPT[action_data.get("chapter", 1)]
            summon = self.parent.parent.process_summon(summon)
            uuid = f"Compendium.icon.foes.Actor.{summon.id}"
            summons.append(uuid)

        # Build the primary choice
        primary_choice = {
            "name": action_data.get("name"),
            "actions": action_data.get("action_cost", 0),
            "round_action": action_data.get("round_action", False),
            "tags": other_tags,
            "ranges": ranges,
            "trigger": action_data.get("trigger", ""),
            "combo": 0,
            "resolve": 0,
            "effects": effects,
            "summons": summons
        }
        return [primary_choice] + choices

    def process_as_ability(self):
        # Process our main data as an action
        self.type = "ability"

        self.system["choices"] = self.action_to_choices(self.data)
        self.system["chapter"] = self.data.get("chapter", 1)
        self.system["special_requirements"] = (
            self.data.get("special_requirements", []),
        )
        self.system["trait"] = False
        self.finalize()

    def process_as_trait(self):
        # Traits follow the same format as actions, actually
        self.type = "ability"
        choices = []
        if self.data.get("description"):
            description = combine_list_desc(self.data.get("description", "Unknown"), self.data.get("listed_items"))
            choices.append({
                "actions": None,
                "round_action": False,
                "tags": [],
                "ranges": [],
                "trigger": self.data.get("trigger", ""),
                "combo": 0,
                "resolve": 0,
                "effects": [description],
            })
        for action in mandate_list(self.data.get("actions")):
            choices += self.action_to_choices(action)

        # Give all of them the Trait tag
        for choice in choices:
            choice["tags"] = ["Trait"] + choice["tags"]

        self.system["choices"] = choices
        self.system["chapter"] = self.data.get("chapter", 1)
        self.system["special_requirements"] = (
            self.data.get("special_requirements", []),
        )
        self.system["trait"] = True
        self.finalize()

    def finalize(self):
        """
        Create the actual document model and write it to a file
        """
        item_sort = len(self.parent.item_ids) + 1
        self.parent.item_ids.append(self.id)
        item = {
            "name": self.name,
            "type": self.type,
            "_id": self.id,
            "system": self.system,
            "img": "icons/svg/combat.svg",
            "sort": item_sort,
            "ownership": {},
            "effects": [],
            "flags": {},
            "_key": f"!actors.items!{self.parent.id}.{self.id}",
        }
        item_filename = f"{self.parent.name}_{self.name}_{self.id[:3]}.json".replace(" ", "_")
        with open(pack_root / "foes" / "_source" / item_filename, "w") as f:
            f.write(json.dumps(item, indent=4))

def main():
    # Post process to combine conditional abilities
    p = Processor()
    p.ingest_all_raw()
    p.process()

    # Convert actions into appropriate items

main()


#!/usr/bin/env python3

import argparse
import numpy as np

from functools import reduce

def iterativeSolver(npData):
    result = 0
    for y in range(len(npData)):
        for x in range(len(npData[y])):
            for direction in [(0, 1), (1, 0), (1, 1), (1, -1)]:
                prod, coord = npData[y][x], (y, x)
                for i in range(1, 4):
                    coord = coord[0] + direction[0], coord[1] + direction[1]
                    if coord[0] >= len(npData) or \
                       coord[0] < 0 or \
                       coord[1] >= len(npData[y]) or \
                       coord[1] < 0:
                        prod = 0
                        break
                    else:
                        print(coord, npData[coord[0]][coord[1]])
                        prod *= npData[coord[0]][coord[1]]
                if prod > result:
                    result = prod
                print(result)
                print("____")
    return result

def functionalSolver(grid):
    return max(
        [
            (
                reduce(
                    (lambda a, b: a*b),
                    [
                        grid[y + n*direction[0]][x + n*direction[1]]
                        for n in range(4) if y + n*direction[0] < len(grid)
                        and y+n*direction[0] >= 0
                        and x+n*direction[1] < len(grid[y])
                        and x+n*direction[1] >= 0
                    ]
                )
            )
            for y in range(len(grid))
            for x in range(len(grid[y]))
            for direction in ((0, 1), (1, 0), (1, 1), (1, -1))])

def solve(dataFile):
    data = np.loadtxt(dataFile, dtype=int)
    print(functionalSolver(data))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--file',
        help='NxN grid of ints',
        required=True
    )
    args = parser.parse_args()

    solve(args.file)

if __name__ == "__main__":
    main()


import tensorflow as tf

tf.compat.v1.disable_eager_execution()

import argparse
import cv2
import importlib
import numpy as np
import os

import data

from distutils.util import strtobool
from tensorflow.keras.optimizers import Adam
import tensorflow.keras.metrics as keras_metrics

from models.available_models import get_models_dict
from callbacks_and_losses import custom_losses

# Used for memory error in RTX2070
physical_devices = tf.config.experimental.list_physical_devices('GPU')
config = tf.config.experimental.set_memory_growth(physical_devices[0], True)

models_dict = get_models_dict()

def get_weight_paths(folder_path):
    models = sorted([f for f in os.listdir(folder_path)
                     if not f.startswith(".") and os.path.isdir(os.path.join(folder_path, f))],
                    key=lambda f: f.lower())
    return [os.path.join(folder_path, model, "results_training_min_val_loss", "%s_best.hdf5" % model) for model in
            models]

def majority_voting(predictions):
    voting = np.sum(predictions, axis=-1) / predictions.shape[-1]
    return np.where(voting > 0.5, 1.0, 0.0)

def consensus_voting(predictions):
    voting = np.sum(predictions, axis=-1)
    return np.where(voting >= float(predictions.shape[-1]), 1.0, 0.0)

def main(args):
    weights = get_weight_paths(args.weights_folder)

    # Here we find to paths to all images from the selected datasets
    image_paths = data.create_image_paths(args.dataset_names, args.dataset_paths)

    input_size = (None, None)

    model = models_dict[args.model]((input_size[0], input_size[1], 1))
    try:
        # Model name should match with the name of a model from
        # https://www.tensorflow.org/api_docs/python/tf/keras/applications/
        # This assumes you used a model with RGB inputs as the first part of your model,
        # therefore your input data should be preprocessed with the corresponding
        # 'preprocess_input' function
        m = importlib.import_module('tensorflow.keras.applications.%s' % model.name)
        rgb_preprocessor = getattr(m, "preprocess_input")
    except ModuleNotFoundError:
        rgb_preprocessor = None

    model.compile(optimizer=Adam(), loss=custom_losses.bce_dsc_loss(),
                  metrics=[custom_losses.dice_coef, 'binary_crossentropy',
                           keras_metrics.Precision(), keras_metrics.Recall()])

    save_to = "results_%s_%s" % ("-".join(args.dataset_names), args.voting_strategy)
    if not os.path.exists(save_to):
        os.makedirs(save_to)
    i = 0
    n = image_paths.shape[1]
    for path in image_paths.transpose():
        print("\rProcessed {}%".format(round(i*100 / n, 1)), end='')
        name = os.path.split(path[1])[-1]
        name, extension = os.path.splitext(name)
        extension = ".png"

        predictions = []
        for m in range(len(weights)):
            model.load_weights(weights[m])
            [im, gt, pred] = data.test_image_from_path(model, path[0], None)
            or_shape = cv2.imread(path[0], cv2.IMREAD_GRAYSCALE).shape
            pred = pred[:or_shape[0], :or_shape[1]]
            predictions.append(np.where(pred > 0.5, 1.0, 0.0))

        decision = majority_voting(
            np.concatenate(predictions, axis=-1)) if args.voting_strategy == "majority" else consensus_voting(
            np.concatenate(predictions, axis=-1))
        cv2.imwrite(os.path.join(save_to, name + extension), 255 * decision)
        i += 1
    print("\rProcessed {}%".format(round(i * 100 / n, 1)))

def parse_args(args=None):
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--dataset_names", type=str, nargs="+",
                        help="Must be one of: 'cfd', 'cfd-pruned', 'aigle-rn', 'esar', 'crack500', 'gaps384', "
                             "'cracktree200', 'text'")
    parser.add_argument("-p", "--dataset_paths", type=str, nargs="+",
                        help="Path to the folders or files containing the respective datasets as downloaded from the "
                             "original source.")
    parser.add_argument("-w", "--weights_folder", type=str,
                        help="Path to folder generated by 'train_for_bagging'.")
    parser.add_argument("-m", "--model", type=str, default="uvgg19",
                        help="Network to use. It can be either a name from 'models.available_models.py' or a path to a "
                             "json file.")
    parser.add_argument("-s", "--voting_strategy", type=str, default="majority",
                        help="Either 'majority' or 'consensus' voting..")

    args_dict = parser.parse_args(args)
    for attribute in args_dict.__dict__.keys():
        if args_dict.__getattribute__(attribute) == "None":
            args_dict.__setattr__(attribute, None)
        if args_dict.__getattribute__(attribute) == "True" or args_dict.__getattribute__(attribute) == "False":
            args_dict.__setattr__(attribute, bool(strtobool(args_dict.__getattribute__(attribute))))
    return args_dict

if __name__ == "__main__":
    args = parse_args()
    main(args)


import sys
import numpy as np
import cv2
import os
from sys import platform
import argparse
import time
import tkinter as tk
from multiprocessing import Value, Array, Process
import hand_module as hm
import pyxel

root = tk.Tk()
root.title("認識結果")
root.geometry("600x600+1200+50")

def changechannel(now,x):
    if x == 1:
        now+=1
    if x == -1:
        now-=1
    return now%8

def channelToImage(c):
    if c == 1:
        return 8
    if c == 2:
        return 3
    if c == 3:
        return 8
    if c == 4:
        return 9
    if c == 5:
        return 14
    if c == 6:
        return 12
    if c == 7:
        return 10
    if c == 8:
        return 6
    if c == 9:
        return 2
def volumeChange(vol,x):
    if x == 1:
        if vol<30:
            vol += 1

    if x == -1:
        if 0<vol:
            vol -= 1

    return vol

class TV:
    def __init__(self,onoff,vol,volcon,now):
        self.channel = [1,2,4,5,6,7,8,9]
        self.volcon = volcon
        self.range = 0
        self.onoff = onoff
        print(type(self.onoff))
        self.vol = vol
        self.now = now
        pyxel.init(60, 45)
        pyxel.run(self.update,self.draw)

    def update(self):

        if self.onoff.value == 1:

            if pyxel.btnr(pyxel.KEY_RIGHT):
                self.now.value = changechannel(self.now.value,1)
            if pyxel.btnr(pyxel.KEY_LEFT):
                self.now.value = changechannel(self.now.value,-1)
            if pyxel.btnr(pyxel.KEY_UP):
                pyxel.cls(channelToImage(self.channel[self.now.value]))
                pyxel.text(int(pyxel.width)-5, 5, str(self.channel[self.now.value]), 0)
                self.vol.value = volumeChange(self.vol.value,1)
                self.volcon.value = 50

            if pyxel.btnr(pyxel.KEY_DOWN):
                pyxel.cls(channelToImage(self.channel[self.now.value]))
                pyxel.text(int(pyxel.width)-5, 5, str(self.channel[self.now.value]), 0)
                self.vol.value = volumeChange(self.vol.value,-1)
                self.volcon.value = 50

        if pyxel.btnr(pyxel.KEY_O) and self.onoff.value==1:
            self.onoff.value = False
            self.range = int(pyxel.height/2)
            self.volcon.value = 0

        if pyxel.btnr(pyxel.KEY_I) and self.onoff.value==0:
            self.onoff.value = True
            self.range = 0
            print(self.onoff.value)

    def draw(self):

        if self.onoff.value == 1:
            pyxel.cls(channelToImage(self.channel[self.now.value]))
            if self.range < int(pyxel.height/2):
                pyxel.rect(0, int(pyxel.height/2)-self.range, pyxel.width, self.range*2, 7)
                self.range += 5
            pyxel.text(5, 5, str(self.channel[self.now.value]), 0)

        else:
            pyxel.cls(0)
            if 0 < self.range:
                pyxel.rect(0, int(pyxel.height/2)-self.range, pyxel.width, self.range*2, 7)
                self.range -= 5

        if self.volcon.value > 0:
            pyxel.text(int(pyxel.width/2), int(pyxel.height)-5, "vol:{}".format(self.vol.value), 0)
            self.volcon.value -= 1

def TVstart(onoff,vol,volcon,now):
    TV(onoff,vol,now)

# Import Openpose (Windows/Ubuntu/OSX)
dir_path = os.path.dirname(os.path.realpath(__file__))
try:
    # Windows Import
    if platform == "win32":
        # Change these variables to point to the correct folder (Release/x64 etc.)
        sys.path.append(dir_path + '/../../python/openpose/Release');
        os.environ['PATH']  = os.environ['PATH'] + ';' + dir_path + '/../../x64/Release;' +  dir_path + '/../../bin;'
        import pyopenpose as op
    else:
        # Change these variables to point to the correct folder (Release/x64 etc.)
        sys.path.append('../../python');
        # If you run `make install` (default path is `/usr/local/python` for Ubuntu), you can also access the OpenPose/python module from there. This will install OpenPose and the python library at your desired installation path. Ensure that this is in your python path in order to use it.
        # sys.path.append('/usr/local/python')
        from openpose import pyopenpose as op
except ImportError as e:
    print('Error: OpenPose library could not be found. Did you enable `BUILD_PYTHON` in CMake and have this Python script in the right folder?')
    raise e

def openpose_demo(onoff,vol,volcon,now):
    # Custom Params (refer to include/openpose/flags.hpp for more parameters)
    params = dict()
    params["model_folder"] = "../../../models/"
    params["hand"] = True

    # Starting OpenPose
    opWrapper = op.WrapperPython()
    opWrapper.configure(params)
    opWrapper.start()

    cap = cv2.VideoCapture(1) #0だったり1だったり
    #cap.set(3,1280)
    #cap.set(4,960)
    cap.set(3,1920)
    cap.set(4,1080)

    before_form = []
    current_form = []
    counter = 0
    while(True):
        ret,frame = cap.read()

        # Process Image
        datum = op.Datum()
        imageToProcess = frame
        datum.cvInputData = imageToProcess

        opWrapper.emplaceAndPop([datum])

        cv2.namedWindow("OpenPose 1.5.0 - Tutorial Python API", cv2.WINDOW_KEEPRATIO | cv2.WINDOW_NORMAL) #ウィンドウサイズを可変に
        cv2.imshow("OpenPose 1.5.0 - Tutorial Python API", datum.cvOutputData)

        #print(datum.handKeypoints[1][0])
        try:
            right_hand,flag = hm.is_hand_recog(datum.handKeypoints[1][0])
        except Exception as e:
            continue
        #カメラの指定によっては「IndexError: too many indices for array」というエラーが出る

        if flag == True:
            current_form = hm.check_handform2(right_hand)
            print(current_form)
            if current_form == before_form:   #1フレーム前の形と現在の形を比較する
                counter = counter + 1  #同じだったらカウントを１増やす
                if(counter == 5): #カウントが10になったら（10回連続して同じ形を認識したら）
                    n = hm.list_to_num(current_form) #手の形から番号を決定
                    print(n)
                    if n == '5':
                        if onoff.value == 1:
                            onoff.value = 0
                        else:
                            onoff.value = 1
                    if n == '4':
                        now.value = changechannel(now.value,1)
                        counter = 0
                    if n == '3':
                        now.value = changechannel(now.value,-1)
                        counter = 0
                    if n == '2':
                        vol.value = volumeChange(vol.value,1)
                        volcon.value = 50
                        counter = 0
                    if n == '1':
                        vol.value = volumeChange(vol.value,-1)
                        volcon.value = 50
                        counter = 0
                    if counter == 10:
                        counter = 0

            else:
                counter = 0 #違ったらカウントをリセットする
            before_form = current_form
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    cap.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    onoff = Value('i', 0)
    vol = Value('i', 10)
    volcon = Value('i', 0)
    now = Value('i', 0)

    th1 = Process(target=openpose_demo,args=[onoff,vol,volcon,now])
    #th2 = Process(target=TVstart,args=[onoff,vol,now])
    th1.start()
    #th2.start()
    TV(onoff,vol,volcon,now)


from timeit import repeat
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import random as rand
#BOILERPLATE
plt.style.use('dark_background')
fig = plt.figure()
plt.get_current_fig_manager().window.state('zoomed')
ax = fig.add_subplot(111)
ax.set_aspect( 'auto' )
ax.axis('square')
ax.set_xlim( [ -4e8, 4e8 ] )
ax.set_ylim( [ -4e8, 4e8 ] )
#CONSTANTS
G = 6.67e-11
T_SPAN = 60 * 60 * 24 * 10 # seconds
SPEED = 10 # times faster
n = 0
offset = 10
delta_t = 150 # seconds
STEPS = int(T_SPAN / delta_t)

class Body:
    def __init__(self, rx, ry, vx, vy, mass, color):
        self.rx = rx
        self.ry = ry
        self.vx = vx
        self.vy = vy
        self.mu = G*mass
        self.color = color
        global n
        n+=1
    def state_vectors(self):
        return [self.rx, self.ry, self.vx, self.vy]
# bodies = [
#     #JOOL
#     Body(0,0,0,0,4.2332e24,'green'),
#     #LAYTHE
#     Body(0, -27184000, 3224 , 0, 2.9397e22, 'blue'),
#     #VALL
#     Body(0,-43152000, 2559, 0, 3.1087655e21,'cyan'),
#     #TYLO
#     Body(0,-68500000, 2031, 0, 4.2332127e22,'grey'),
#     #BOP
#     Body(0,-98302500, 1884, 0, 3.726109e19,'brown'),
#     #POL
#     Body(0,-149155794, 1489, 0, 1.0813507e19,'yellow')
# ]
bodies=[]
colours=["blue", "red", "green", "yellow", "white", "purple", "brown", "grey", "orange", "violet"]
for i in range(rand.randint(5,12)):
    bodies.append(Body(rand.randint(-4e8,4e8), rand.randint(-4e8,4e8), rand.randint(-5000,5000), rand.randint(-5000,5000), rand.randint(5e17, 5e24), colours[rand.randint(0, len(colours)-1)]))
states = np.zeros((STEPS, n, 4))
coords = np.zeros((STEPS, n, 2))
# initialize initial state vectors
for i in range(n):
    states[0][i]=bodies[i].state_vectors()
def acceleration(state):
    a = np.zeros((n, 2))
    for x in range(n):
        for i in range(n):
            if (x == i):
                continue
            ri = np.array([state[i][0], state[i][1]])
            rx = np.array([state[x][0], state[x][1]])
            a[x]+=bodies[i].mu*(ri - rx)/(np.linalg.norm(ri-rx))**3
    # Derivative States
    d_states = np.zeros((n, 4))
    for i in range(n):
        d_states[i][0] = state[i][2]
        d_states[i][1] = state[i][3]
        d_states[i][2] = a[i][0]
        d_states[i][3] = a[i][1]
    return d_states

def integrate( f, y, dt ):
    global delta_t
    # Current Step Size
    k1 = f(y )
    k2 = f(y + 0.5 * k1 * dt )
    k3 = f(y + 0.5 * k2 * dt )
    k4 = f(y +       k3 * dt )
    next_step = y + dt / 6.0 * ( k1 + 2 * k2 + 2 * k3 + k4 )
    return next_step

for step in range(STEPS - 1):
    states[step+1]=integrate(acceleration, states[step], delta_t)

coordsX = [[] for _ in range(n)]
coordsY = [[] for _ in range(n)]
cnt = 0
for state in states:
    for j in range(n):
        if cnt % SPEED == 0:
            coordsX[j].append(state[j][0])
            coordsY[j].append(state[j][1])
    cnt+=1
lines = [plt.plot([], [])[0] for _ in range(2*n)]
def init():
    for i in range(n):
        lines[i].set_data([], [])
        lines[i+n].set_data([],[])
        lines[i].set_color(bodies[i].color)
        lines[i+n].set_color(bodies[i].color)
        lines[i].set_linewidth(1)
        lines[i+n].set_marker('.')
    return lines
def animate(i):
    try:
        for j in range(n):
            if i >= offset:
                lines[j].set_data(coordsX[j][i-offset:i], coordsY[j][i-offset:i])
            else:
                lines[j].set_data(coordsX[j][:i], coordsY[j][:i])
            lines[j+n].set_data(coordsX[j][i], coordsY[j][i])
        return lines
    except IndexError:
        i = 0
        return lines
ani = FuncAnimation(fig, animate, init_func=init, interval=30, blit=True, repeat=True, frames=len(coordsX[0]))
print(len(coordsX[0]), len(coordsX[1]), len(coordsX[2]), len(coordsX[3]))
plt.show()


import ast, getopt, sys, copy, os
from fractions import Fraction

clear = lambda: os.system('cls' if os.name == 'nt' else 'clear')

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKCYAN = '\033[96m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

class SimplexSolver():
    ''' Solves linear programs using simplex algorithm and
        output problem on terminal.
    '''

    def __init__(self):
        self.A = []
        self.b = []
        self.c = []
        self.tableau = []
        self.entering = []
        self.departing = []
        self.ineq = []
        self.prob = "max"

    def run_simplex(self, A, b, c, prob='max', ineq=[],
                    enable_msg=False):
        ''' Run simplex algorithm.
        '''
        self.prob = prob
        self.ineq = ineq

        # Add slack & artificial variables
        self.set_simplex_input(A, b, c)

        # Are there any negative elements on the bottom (disregarding
        # right-most element...)
        while (not self.should_terminate()):
            # ... if so, continue.
            if(enable_msg):
                clear()
                self._print_tableau()
                print("Solucao atual: %s\n" %
                      str(self.get_current_solution()))
                self._prompt()

            # Attempt to find a non-negative pivot.
            pivot = self.find_pivot()
            if pivot[1] < 0:
                if (enable_msg):
                    print ("Nao existe um pivot negativo. "
                           "Portanto, a solucao e impratica.")
                return None
            else:
                if (enable_msg):
                    clear()
                    self._print_tableau()
                    print((f"\nComo ainda temos elementos negativos na linha de baixo, "
                          "entao ainda temos que continuar o algoritmo. "
                          "Entao, pivotaremos a solucao atual. A "
                          "variavel de entrada e %s %s %s e a variavel de saida "
                          "da base e %s %s %s.\n" %
                           (bcolors.OKGREEN,
                            str(self.entering[pivot[0]]),
                            bcolors.ENDC,
                            bcolors.FAIL,
                            str(self.departing[pivot[1]]),
                            bcolors.ENDC)))
                    self._prompt()
                    print("\nPerform elementary row operations until the "
                          "pivot is one and all other elements in the "
                          "entering column are zero.\n")

            # Do row operations to make every other element in column zero.
            self.pivot(pivot)

        solution = self.get_current_solution()
        if (enable_msg):
            clear()
            self._print_tableau()
            print(("Solucao atual: %s\n" % str(solution)))
            print("Finalizando...")
        return solution

    def set_simplex_input(self, A, b, c):
        ''' Set initial variables and create tableau.
        '''
        # Convert all entries to fractions for readability.
        for a in A:
            self.A.append([Fraction(x) for x in a])
        self.b = [Fraction(x) for x in b]
        self.c = [Fraction(x) for x in c]
        if not self.ineq:
            if self.prob == 'max':
                self.ineq = ['<='] * len(b)
            elif self.prob == 'min':
                self.ineq = ['>='] * len(b)

        self.update_enter_depart(self.get_Ab())

        # If this is a minimization problem...
        if self.prob == 'min':
            # ... find the dual maximum and solve that.
            m = self.get_Ab()
            m.append(self.c + [0])
            m = [list(t) for t in zip(*m)] # Calculates the transpose
            self.A = [x[:(len(x)-1)] for x in m]
            self.b = [y[len(y) - 1] for y in m]
            self.c = m[len(m) -1]
            self.A.pop()
            self.b.pop()
            self.c.pop()
            self.ineq = ['<='] * len(self.b)

        self.create_tableau()
        self.ineq = ['='] * len(self.b)
        self.update_enter_depart(self.tableau)

    def update_enter_depart(self, matrix):
        self.entering = []
        self.departing = []
        # Create tables for entering and departing variables
        for i in range(0, len(matrix[0])):
            if i < len(self.A[0]):
                prefix = 'x' if self.prob == 'max' else 'y'
                self.entering.append("%s_%s" % (prefix, str(i + 1)))
            elif i < len(matrix[0]) - 1:
                self.entering.append("s_%s" % str(i + 1 - len(self.A[0])))
                self.departing.append("s_%s" % str(i + 1 - len(self.A[0])))
            else:
                self.entering.append("b")

    def add_slack_variables(self):
        ''' Add slack & artificial variables to matrix A to transform
            all inequalities to equalities.
        '''
        slack_vars = self._generate_identity(len(self.tableau))
        for i in range(0, len(slack_vars)):
            self.tableau[i] += slack_vars[i]
            self.tableau[i] += [self.b[i]]

    def create_tableau(self):
        ''' Create initial tableau table.
        '''
        self.tableau = copy.deepcopy(self.A)
        self.add_slack_variables()
        c = copy.deepcopy(self.c)
        for index, value in enumerate(c):
            c[index] = -value
        self.tableau.append(c + [0] * (len(self.b)+1))

    def find_pivot(self):
        ''' Find pivot index.
        '''
        enter_index = self.get_entering_var()
        depart_index = self.get_departing_var(enter_index)
        return [enter_index, depart_index]

    def pivot(self, pivot_index):
        ''' Perform operations on pivot.
        '''
        j,i = pivot_index

        pivot = self.tableau[i][j]
        self.tableau[i] = [element / pivot for
                           element in self.tableau[i]]
        for index, row in enumerate(self.tableau):
           if index != i:
              row_scale = [y * self.tableau[index][j]
                          for y in self.tableau[i]]
              self.tableau[index] = [x - y for x,y in
                                     zip(self.tableau[index],
                                         row_scale)]

        self.departing[i] = self.entering[j]

    def get_entering_var(self):
        ''' Get entering variable by determining the 'most negative'
            element of the bottom row.
        '''
        bottom_row = self.tableau[len(self.tableau) - 1]
        most_neg_ind = 0
        most_neg = bottom_row[most_neg_ind]
        for index, value in enumerate(bottom_row):
            if value < most_neg:
                most_neg = value
                most_neg_ind = index
        return most_neg_ind

    def get_departing_var(self, entering_index):
        ''' To calculate the departing variable, get the minimum of the ratio
            of b (b_i) to the corresponding value in the entering collumn.
        '''
        skip = 0
        min_ratio_index = -1
        min_ratio = 0
        for index, x in enumerate(self.tableau):
            if x[entering_index] != 0 and x[len(x)-1]/x[entering_index] > 0:
                skip = index
                min_ratio_index = index
                min_ratio = x[len(x)-1]/x[entering_index]
                break

        if min_ratio > 0:
            for index, x in enumerate(self.tableau):
                if index > skip and x[entering_index] > 0:
                    ratio = x[len(x)-1]/x[entering_index]
                    if min_ratio > ratio:
                        min_ratio = ratio
                        min_ratio_index = index

        return min_ratio_index

    def get_Ab(self):
        ''' Get A matrix with b vector appended.
        '''
        matrix = copy.deepcopy(self.A)
        for i in range(0, len(matrix)):
            matrix[i] += [self.b[i]]
        return matrix

    def should_terminate(self):
        ''' Determines whether there are any negative elements
            on the bottom row
        '''
        result = True
        index = len(self.tableau) - 1
        for i, x in enumerate(self.tableau[index]):
            if x < 0 and i != len(self.tableau[index]) - 1:
                result = False
        return result

    def get_current_solution(self):
        ''' Get the current solution from tableau.
        '''
        solution = {}
        for x in self.entering:
            if x is not 'b':
                if x in self.departing:
                    solution[x] = self.tableau[self.departing.index(x)]\
                                  [len(self.tableau[self.departing.index(x)])-1]
                else:
                    solution[x] = 0
        solution['z'] = self.tableau[len(self.tableau) - 1]\
                          [len(self.tableau[0]) - 1]

        # If this is a minimization problem...
        if (self.prob == 'min'):
            # ... then get x_1, ..., x_n  from last element of
            # the slack columns.
            bottom_row = self.tableau[len(self.tableau) - 1]
            for v in self.entering:
                if 's' in v:
                    solution[v.replace('s', 'x')] = bottom_row[self.entering.index(v)]

        return solution

    def _generate_identity(self, n):
        ''' Helper function for generating a square identity matrix.
        '''
        I = []
        for i in range(0, n):
            row = []
            for j in range(0, n):
                if i == j:
                    row.append(1)
                else:
                    row.append(0)
            I.append(row)
        return I

    def _print_matrix(self, M):
        ''' Print some matrix.
        '''
        for row in M:
            print('|', end=' ')
            for val in row:
                print('{:^6}'.format(str(val)), end=' ')
            print('|')

    def _print_tableau(self):
        ''' Print simplex tableau.
        '''
        print(' ', end=' ')
        for val in self.entering:
            print('{:^6}'.format(str(val)), end=' ')
        print(' ')
        for num, row in enumerate(self.tableau):
            print('|', end=' ')
            for index, val in enumerate(row):
                print('{:^6}'.format(str(val)), end=' ')
            if num < (len(self.tableau) -1):
                print('| %s' % self.departing[num])
            else:
                print('|')

    def _prompt(self):
        input("Press enter to continue...")

if __name__ == '__main__':
    clear()

    ''' COMMAND LINE INPUT HANDLING '''
    A = []
    b = []
    c = []
    p = ''
    argv = sys.argv[1:]
    try:
        opts, args = getopt.getopt(argv,"hA:b:c:p:",["A=","b=","c=","p="])
    except getopt.GetoptError:
        print('simplex.py -A <matrix> -b <vector> -c <vector> -p <type>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print('simplex.py -A <matrix> -b <vector> -c <vector> -p <obj_func_type>')
            print('A: Matrix that represents coefficients of constraints.')
            print('b: Ax <= b')
            print('c: Coefficients of objective function.')
            print('p: Indicates max or min objective function.')
            sys.exit()
        elif opt in ("-A"):
            A = ast.literal_eval(arg)
        elif opt in ("-b"):
            b = ast.literal_eval(arg)
        elif opt in ("-c"):
            c = ast.literal_eval(arg)
        elif opt in ("-p"):
            p = arg.strip()
    if not A or not b or not c:
        print('Must provide arguments for A, b, c (use -h for more info)')
        sys.exit()
    ''' END OF COMMAND LINE INPUT HANDLING '''

    # Assume maximization problem as default.
    if p not in ('max', 'min'):
        p = 'max'

    SimplexSolver().run_simplex(A,b,c,prob=p,enable_msg=True)


import numpy as np
import pandas as pd
from geopy.distance import geodesic
from dateutil.parser import parse

def trajcheck(file, lat_diff=1., lon_diff=1., search=None, path = '/home/ollie/muali/Data/winter_all/'):
    """This function checks if a given trajectory is passing through a observation station.
    Outputs the matching hour, Obs Station Name, etc.
    Modified to give distance in km between the traj and station as well

    file_= Name of the time from winter_all folder as'tdump_lvl_MM_DD_HH'

    lat_diff= Minimum latitude difference to satisfy, default 1.0

    lon_diff =Minimum latitude difference to satisfy, default 1.0

    search: (str) to be used for searching a pattern from the output

    path: default is winter_all folder """

    file_name = path + file

    df_IGR = pd.read_excel('/home/ollie/muali/python_notebooks/IGR_25N_v2.xlsx')
    #f = open(file_name,'rb+')

    # getting launch date
    #for i, file_ in enumerate(f):
     #   if i ==4: # reads 6th line
      #      date_str = f.read(25)
       #     f.close()
    #l_date = parse(date_str)

    df = pd.read_csv(file_name, skiprows=7, header=None, delim_whitespace=True)
    # a very efficient way to rename columns
    # dict to rename columns
    cols_renames = {2: 'year', 3: 'month', 4: 'day', 5: 'hour', 6: 'minute'}

    # converting year to 4 digits
    df.iloc[:,2] = df.iloc[:,2] + 1900

    # df_subset is used to merge yy, mm, dd, hh values into BT_time column
    df_subset = df.loc[:, list(cols_renames.keys())].rename(columns=cols_renames)
    # df_subset is extracting the time from the original dataframe to be used to make a time index for the final dataframe
    dt_series = pd.to_datetime(df_subset) # converts the df into a time series which we will add as extra column
    dt_series.head()
    # adding a new backtrajectory column
    df['BT_time'] = dt_series # adding the back trajectory time column to df

    # dropping the not required columns
    df.drop([0,1,2,3,4,5,6,7], axis =1, inplace=True)
    df.head()

    col_rename2 = {8: 'back_hr', 9:'Lat', 10: 'Lon', 11:'AGL', 12:'Pressure'}
    df.rename(columns=col_rename2, inplace=True)

    del df_subset
    del dt_series

    count = 0
    # count will be used to tell how many times our condition is met

    string_list =[] # to store the output of print to be used by regex
    for row_IRG in df_IGR.itertuples():
        for row in df.itertuples():
            # lat_diff lon_diff provided by the user
            if ((-lat_diff < (row_IRG.Lat - row.Lat) < lat_diff) & (-lon_diff < (row_IRG.Lon - row.Lon) < lon_diff)):
              #  print(row.back_hr, row.BT_time, row_IRG.ID, row_IRG.Lat, row_IRG.Lon ,row_IRG.Lat - row.Lat, row_IRG.Lon - row.Lon)
#                print(row.back_hr, row.BT_time.strftime('%d-%m-%Y %H:%M'), row_IRG.ID, row_IRG.Lat, row_IRG.Lon , row.Lat - row_IRG.Lat, row.Lon - row_IRG.Lon, geodesic((row.Lat, row.Lon), (row_IRG.Lat, row_IRG.Lon)).km )
                string_list.append((row.back_hr, row.BT_time.strftime('%d-%m-%Y %H:%M'), row_IRG.ID, row_IRG.Lat, row_IRG.Lon , row.Lat - row_IRG.Lat, row.Lon - row_IRG.Lon, geodesic((row.Lat, row.Lon), (row_IRG.Lat, row_IRG.Lon)).km ))
                count = count + 1

    print("The following are the counts:")
    print(count)

    if search is not None:
        for item in string_list:
            if search in item:
                print(item,'\n')

    else:
        for item in string_list:
            print(item,'\n')
# eof


import json
from pathlib import Path
from collections import defaultdict

counts = defaultdict(int)

def walk(root: Path):
    for thing in root.iterdir():
        if thing.is_dir():
            walk(thing)
        elif thing.name == "index.json":
            with open(thing) as f:
                doc = json.load(f)["doc"]
                flaws = doc.get("flaws") or {}
                for flawgroup in flaws:
                    if flawgroup == "macros":
                        for macroflaw in flaws[flawgroup]:
                            group = f"macros__{macroflaw['name']}"
                            counts[group] += 1
                    else:
                        counts[flawgroup] += 1

walk(Path("client/build/en-us/docs/"))

# flat = sorted[(v,k) for k,v in counts.items()]
# from pprint import pprint
# pprint(counts)
for flaw in sorted(counts):
    print(flaw.ljust(35), f"{counts[flaw]:,}")


def DownloadYoutubeVid(link,SAVE_PATH = "D:\Hajer\Scrapper\ytVideos"    ):
    from pytube import YouTube
    try:
        # object creation using YouTube
        # which was imported in the beginning
        yt = YouTube(link)
    except:
        print("Connection Error") #to handle exception
    try:
        did=False
        a=yt.streams.filter(res="1080p")
        if list(a)==[]:
            a=yt.streams.filter(res="720p")
            if list(a)!=[]:
                a.first().download(output_path = SAVE_PATH, filename = yt.title)
                did=True
        else:
            a.first().download(output_path = SAVE_PATH, filename = yt.title)
            did=True
        if not(did)&(list(yt.streams)!=[]):
            yt.streams.first().download(SAVE_PATH, filename = yt.title)
    except:
        print("Some Error!")
    #os.rename(yt.streams.first().default_filename, 'new_filename.ext')

def getlinks(csvpath="D:\Hajer\Scrapper\ml-youtube.csv"):
    import csv
    links=[]
    with open(csvpath, mode='r',encoding="utf8") as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        #csv_reader = csv.DictReader(csv_file)
        line_count = 0
        for row in csv_reader:
            if line_count == 0:
                #print(f'Column names are {", ".join(row)}')
                line_count += 1
                continue
            #print(f'\t{row[0]} ---- {row[1]} ---- {row[2]}.')
            link="https://www.youtube.com/watch?v="+row[0]
            links.append(link)
            line_count += 1
        print(f'Processed {line_count} lines.')
    return links

DownloadYoutubeVid("https://www.youtube.com/watch?v=Q5s_HCPdD6E")
#links=getlinks(csvpath="D:\Hajer\Scrapper\ml-youtube.csv")
# for link in links:
#     DownloadYoutubeVid(link)


import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pickup_map.settings.local')
import django
django.setup()
from main.models import Point
from django.shortcuts import get_object_or_404

def connectPoints(point, *points):
    pt = get_object_or_404(Point, pk=point) # 포인트 노드를 찾음
    for i in points:
        x = get_object_or_404(Point, pk=i) # 연결할 포인트
        data = str(point)+","   # 연결할 포인트에 현재 포인트 추가
        x.connectPoints += data
        x.save()

        data = str(i)+","   # 포인트에 연결할 포인트를 추가
        pt.connectPoints += data
    pt.save()

def connectMarketAndPoint(market, *points):
    mk = get_object_or_404(Point, pk=market)
    for i in points:
        pt = get_object_or_404(Point, pk=i)
        data = str(market)+","
        pt.connectMarkets += data
        pt.save()

        data = str(i)+","
        mk.connectPoints += data
    mk.save()

def connectPointAndMarket(point, *markets):
    pt = get_object_or_404(Point, pk=point)
    for i in markets:
        data = str(point)+","
        mk = get_object_or_404(Point, pk=i)
        mk.connectPoints += data
        mk.save()
        data = str(i)+","
        pt.connectMarkets += data
    pt.save()

from collections import deque

# BFS 메서드 정의
def bfs(start, end): # 시작 포인트, 끝 포인트
    if start == end: return 1

    visited = [False] * 400

    visited[start] = True
    queue = deque([start])
    distant = 1

    while queue:
        size = len(queue)

        while size:
            v = queue.popleft()
            cur = get_object_or_404(Point, pk=v)

            next_points = [int(i) for i in cur.connectPoints.split(',')[:-1]]
            for i in next_points:
                if i == end: return distant + 1

            for i in next_points:
                if not visited[i]:
                    queue.append(i)
                    visited[i] = True
            size -= 1
        distant += 1
    return distant

# DFS 메서드 정의
done = False

def dfs(v, visited, queue, end, step, distance):
    visited[v] = True
    if step > distance: return

    if v == end and step == distance:
        global done
        done = True
        return

    cur = get_object_or_404(Point, pk=v)
    next_points = [int(i) for i in cur.connectPoints.split(',')[:-1]]

    for i in next_points:
        if not visited[i]:
            dfs(i, visited, queue, end, step+1, distance)
            if done == True:
                queue.append(i)
                return
            visited[i] = False

def toElebator(_from):
    from_store = get_object_or_404(Point, pk=_from)
    start_points = [int(i) for i in from_store.connectPoints.split(',')[:-1]]
    floor = from_store.floor

    end_points = []
    if floor == 1: # 1층일 때
        end_points.extend([116, 117])
    elif floor == 2:
        end_points.extend([27, 28])
    else:
        end_points.extend([36, 37])
    distance = 1000
    point_begin, point_end = 0, 0
    for s in start_points:
        for e in end_points:
            cnt = bfs(s, e)
            if distance > cnt:
                distance = cnt
                point_begin, point_end = s, e

    print(point_begin, point_end, distance)

    visited = [False] * 400
    q = deque()
    global done
    done = False
    dfs(point_begin, visited, q, point_end, 1, distance)
    q.append(point_begin)
    print("경로", list(q)[::-1])
    return list(q)[::-1]

# 상점과 상점 사이의 최소 길을 찾는다. bfs 후 dfs
def findPath(_from, to):
    # 시작 포인트와 끝 포인트를 찾는다.
    from_store = get_object_or_404(Point, pk=_from)
    start_points = [int(i) for i in from_store.connectPoints.split(',')[:-1]]

    to_store = get_object_or_404(Point, pk=to)
    end_points = [int(i) for i in to_store.connectPoints.split(',')[:-1]]

    print(start_points)
    print(end_points)

    if from_store.floor == to_store.floor: # 같은 층일 때
        distance = 1000
        point_begin, point_end = 0, 0
        for s in start_points:
            for e in end_points:
                cnt = bfs(s, e)
                if distance > cnt:
                    distance = cnt
                    point_begin, point_end = s, e

        print(point_begin, point_end, distance)

        visited = [False] * 400
        q = deque()
        global done
        done = False
        dfs(point_begin, visited, q, point_end, 1, distance)
        q.append(point_begin)
        print("경로", list(q)[::-1])
        x = list(q)[::-1]
        l = []
        for i in x:
            pt = get_object_or_404(Point, pk=i)
            l.append([pt.locationX, pt.locationY, pt.locationZ])

        s_point = [from_store.locationX, from_store.locationY, from_store.locationZ]
        e_point = [to_store.locationX, to_store.locationY, to_store.locationZ]
        print(l)
        return x
    else: # 다른 층일 때
        q = toElebator(_from) # 엘리베이터까지 간다
        print(q)
        begin_floor = from_store.floor # 시작 층
        end_floor = to_store.floor # 목적지 층

        begin_elebator = 10
        if begin_floor == 1: begin_elebator = 10
        elif begin_floor == 2: begin_elebator = 20
        else: begin_elebator = 30

        if begin_floor < end_floor: # 올라가는 경우
            while begin_floor != end_floor:
                q.append(begin_elebator)
                begin_elebator += 10
                begin_floor += 1
            q.append(begin_elebator)
        else: # 내려가는 경우
            while begin_floor != end_floor:
                q.append(begin_elebator)
                begin_elebator -= 10
                begin_floor -= 1
            q.append(begin_elebator)
        q.extend(findPath(begin_elebator, to))
        print("q=", q)
        l = []
        for i in q:
            print("i=",i)
            pt = get_object_or_404(Point, pk=i)
            l.append([pt.locationX, pt.locationY, pt.locationZ])

        print(l)
        s_point = [from_store.locationX, from_store.locationY, from_store.locationZ]
        e_point = [to_store.locationX, to_store.locationY, to_store.locationZ]
        # l = 경로들의 좌표
        #
        return q


import json
import os
import requests
from time import sleep
from bs4 import BeautifulSoup

# file_pass = os.path.abspath('rWebDesign.json')
# temporary_data = open(file_pass, 'r')
# json_dict = json.load(temporary_data)#ダブルをシングルに変換

# get_url_info = requests.get(json_dict['data'][1]['from'], headers=headers)
# print(get_url_info.text)

# print(requests.get(json_dict['data'][1]['from']).url)

def get_redirect():
	target_url = 'https://www.pinterest.com/r/pin/554505772865723124/4995915543595742901/4cf0b7ee262acd41d1b0176d1e3ef047f3209215e827b790dc718ca5c1ea0bf6'
	i = 0
	# with requests.Session() as s:
	# 	p = s.post(login_url, data=payload, headers=headers)
		# cookies = p.cookies
		# print(type(cookies))
		# print(json_dict['data'][1]['from'])
	try:
		while 'www.pinterest' in target_url:
			if i > 20 or 'www.pinterest' in target_url and '/r/' not in target_url:
				return target_url

			sleep(0.5)
			r = requests.get(target_url)
			# print(r.url)
			target_url = r.url
			# print(r.url)
			i += 1
		url = r.url
		done = 'これを保存' + url
	except requests.exceptions.ConnectionError as e:
		url = e.request.url
		done = 'これを保存リンク切れ' + url

	print(done)
	return url

def confirmation_url(got_url):
	check_url = got_url

	if 'www.pinterest' in check_url and '/pin/' in check_url and '/r/' not in check_url:
		r = requests.get(check_url)
		html = BeautifulSoup(r.text, 'html.parser')
		head = html.find('head')
		metatag = head.find('meta', attrs={'property':'og:see_also', 'content':True})
		# print(metatag)
		if metatag['content']:
			check_url = metatag['content']

	return check_url

got_url = get_redirect()
got_url = confirmation_url(got_url)
print(got_url)

# result = [login(login_url, payload) for _ in range(9)]; print(result.count(True))

# import requests
# link = 'https://www.pinterest.com/r/pin/554505772869102844/4995915543595742901/ab9290928e62760b540f32156fb9686925897abcbda51275fe2866dd3442d330'
# ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
# headers = {'User-Agent': ua}

# link = requests.get(link, headers=headers)
# print(link.url)


import numpy as np
import utils
import config as c

# Define log class
class Log:
    def __init__(self):
        # Initialize logs
        self.mu = np.zeros((c.n_trials, c.n_steps, c.n_orders, c.n_joints * 3))
        self.mu_dot = np.zeros_like(self.mu)

        self.actions = np.zeros((c.n_trials, c.n_steps, c.n_joints))
        self.actions_dot = np.zeros_like(self.actions)

        self.angles = np.zeros((c.n_trials, c.n_steps, c.n_joints))
        self.est_angles = np.zeros_like(self.angles)

        self.target_pos = np.zeros((c.n_trials, c.n_steps, 2))
        self.est_target_pos = np.zeros_like(self.target_pos)

        self.hand_pos = np.zeros((c.n_trials, c.n_steps, 2))
        self.est_hand_pos = np.zeros_like(self.hand_pos)

        self.mode = np.zeros((c.n_trials, c.n_steps))
        self.int_errors = np.zeros((c.n_trials, c.n_steps, 2, c.n_joints * 3))

    # Track logs for each time step
    def track(self, trial, step, agent, arm, target_pos):
        self.mu[trial, step] = agent.mu
        self.mu_dot[trial, step] = agent.mu_dot

        self.actions[trial, step] = agent.a
        self.actions_dot[trial, step] = agent.a_dot

        limits = np.tile(agent.limits, c.n_joints)
        mu = utils.denormalize(agent.mu[0], limits)
        self.angles[trial, step] = arm.angles
        self.est_angles[trial, step] = mu[:c.n_joints]

        self.target_pos[trial, step] = target_pos
        self.est_target_pos[trial, step] = arm.kinematics(
            mu[c.n_joints: c.n_joints * 2])[-1, :2]

        self.hand_pos[trial, step] = arm.poses[-1, :2]
        self.est_hand_pos[trial, step] = arm.kinematics(
            mu[:c.n_joints])[-1, :2]

        self.mode[trial, step] = agent.mode

        int_errors = agent.E_i * np.repeat(agent.beta, c.n_joints * 3). \
            reshape(2, c.n_joints * 3)
        self.int_errors[trial, step] = int_errors

    # Save log to file
    def save_log(self):
        np.savez_compressed('simulation/log' + c.log_name,
                            mu=self.mu, mu_dot=self.mu_dot,
                            actions=self.actions,
                            actions_dot=self.actions_dot,
                            angles=self.angles,
                            est_angles=self.est_angles,
                            target_pos=self.target_pos,
                            est_target_pos=self.est_target_pos,
                            hand_pos=self.hand_pos,
                            est_hand_pos=self.est_hand_pos,
                            mode=self.mode, int_errors=self.int_errors)


# -*- coding: UTF-8 -*-
#!/usr/bin/python

import logging
import os
import re
import sys

# python 3 porting
# if sys.version_info > (3, 0):
#     import hfst_lookup
#     CHFST = True
# else:
try:
    import hfst_lookup
    CHFST = True
except ImportError:
    from .shared import Alphabet, Header
    from .transducer import Transducer, TransducerW
    CHFST = False

import pymorphy2
from py_tat_morphan.disambiguation import disambiguate_word, load_disam_rules
from pymorphy2.units.by_analogy import (DictionaryAnalyzer,
                                        KnownSuffixAnalyzer,
                                        UnknownPrefixAnalyzer)

logging.basicConfig(filename='log.log', level=logging.DEBUG)

BASE_DIR = os.path.abspath(os.path.dirname(__file__))
DEFAULT_MORPHAN_FILE = os.path.join(BASE_DIR, 'files/tatar_last.hfstol')
WORDS_FILE = os.path.join(BASE_DIR, 'files/words.csv')
TOKENS_FILE = os.path.join(BASE_DIR, 'files/tokens.csv')
EXCEPTIONS_FILE = os.path.join(BASE_DIR, 'files/exceptions.txt')
DISAM_RULES_FILES = os.path.join(BASE_DIR, 'files/disam_rules.json')

PARAMS = {'sdelimiter': u'\t',
          'fdelimiter': u'\n',
          'ignore_newlines': True
         }

def fix(text):
    """
        Prepare text for analysis:
        - Corrects some errors of Tatar language texts
        - Unifies letters, e.g. 'ђ' -> 'ә'
    """
    if sys.version_info < (3, 0) and not isinstance(text, unicode):
        text = text.decode('utf-8')

    letters = {u'ђ':u'ә', u'њ':u'ү', u'ќ':u'җ', u'љ':u'ө', u'ћ':u'ң',
               u'џ':u'һ', u'Ә':u'ә', u'Ү':u'ү', u'Ө':u'ө', u'Җ':u'җ',
               u'Һ':u'һ', u'Ң':u'ң',
               u'Ђ': u'Җ',  #  - почему для ә и Җ используются один и тот де символ?
               u'Љ': u'Ө', u'Њ': u'ү', u'Ќ': u'Җ', u'Џ': u'һ', u'Ћ': u'Ң'}
    # some texts contains words with not right letters, need to replace to right
    for letter in letters:
        text = text.replace(letter, letters[letter])

    # replace some errors
    text = text.replace(u'-\r\n', u'').replace(u'-\n\r', u'')\
               .replace(u'-\n', u'').replace(u'-\r', u'')\
               .replace(u'¬', u'').replace(u'...', u'…')\
               .replace(u'!..', u'!').replace(u'?..', u'?')\
               .replace(u' -', u' - ').replace(u'- ', u' - ')\
               .replace(u'\xad', '').replace(u'\ufeff', '')\
               .replace(u'ª', '').replace(u'’', '')\
               .replace(u'´', '')

    return text

def tokenize(text):
    """
        Splits text into tokens
    """
    tokens = re.split(r"([ .,!?\n\r\t“”„‘«»≪≫\{\}\(\)\[\]:;\'\"+=*\—_^…\|\/\\ ]|[0-9]+)", text)

    # filtering all spaces
    # tokens = filter(lambda token: token not in [u' ', u' ', u'', u'\r', u'\t'], tokens)
    tokens = [token.strip('-') for token in tokens if token not in [' ', u' ', u'', u'\r', u'\t']]

    return tokens

def to_text(sentences, sdelimiter=u'\t', fdelimiter=u'\n'):
    """
        Joins result of morphan into plain text
    """
    result = []
    for sentence in sentences:
        for (token, tag) in sentence:
            result.append('%s%s%s' % (token, sdelimiter, tag))
    return fdelimiter.join(result)

class Morphan:
    def __init__(self,
                 transducerfile=None,
                 wordsfile=None,
                 exceptionsfile=None,
                 disamrulesfile=None,
                 tokensfile=None,
                 params={}):
        '''
            Initialize Morphan:
            - import transducer file from dir
            - import disam files
            - import exception file

        '''
        self.params = PARAMS
        if not transducerfile:
            transducerfile = DEFAULT_MORPHAN_FILE
        if not wordsfile:
            wordsfile = WORDS_FILE
        if not exceptionsfile:
            exceptionsfile = EXCEPTIONS_FILE
        if not disamrulesfile:
            disamrulesfile = DISAM_RULES_FILES
        if not tokensfile:
            tokensfile = TOKENS_FILE
        self.params.update(params)

        # if C hfst_lookup package installed
        if CHFST:
            logging.info('Import CHFST`s Transducer...')
            self.ctransducer = hfst_lookup.Transducer(transducerfile)
            self.transducer = None
        else:
            # else loads the simple transducer
            logging.info('Import Python HFST`s Transducer...')
            self.ctransducer = None
            handle = open(transducerfile, "rb")
            self.header = Header(handle)
            self.alphabet = Alphabet(handle, self.header.number_of_symbols)
            if self.header.weighted:
                self.transducer = TransducerW(handle, self.header, self.alphabet)
            else:
                self.transducer = Transducer(handle, self.header, self.alphabet)
            handle.close()

        # loads words and exceptions for better performance
        self.words = {}
        if not self.ctransducer:
            logging.info('Import Words list...')
            with open(wordsfile, 'rb') as stream:
                lines = stream.read().decode('UTF-8').split('\n')
            for line in lines:
                if not line:
                    continue
                word, morphcode = line.split('\t')
                self.words[word] = morphcode

        logging.info('Import Exceptional words list...')
        with open(exceptionsfile, 'rb') as stream:
            lines = stream.read().decode('UTF-8').split('\n')
            for line in lines:
                if not line:
                    continue
                word, morphcode = line.split('\t')
                self.words[word] = morphcode

        logging.info('Import tokens list...')
        self.tokens = {}
        with open(tokensfile, 'rb') as stream:
            lines = stream.read().decode('UTF-8').split('\n')
            for line in lines:
                if not line:
                    continue
                token, tag = line.split('\t')
                self.tokens[token] = tag

        logging.info('Import Russian Morphan (pymorphy2)...')
        self.rusmorphan = pymorphy2.MorphAnalyzer()

        logging.info('Import Contextual Rules for Disambiguation...')
        self.disam_rules = load_disam_rules(disamrulesfile)

    def lookup(self, token):
        '''
            Loads HFST Trancducer to tag the word
        '''
        if sys.version_info < (3, 0) and not isinstance(token, unicode):
            token = token.decode('UTF-8')

        if token.lower() in self.words:
            return self.words[token.lower()]

        if self.ctransducer:
            # python 3 porting
            if sys.version_info < (3, 0):
                token = token.encode('UTF-8')
            result = self.ctransducer.lookup(token)
            if not result:
                return None

            # delete '+' sign in the end of analysis
            # python 3 porting
            if sys.version_info < (3, 0):
                result = [row[0].decode('UTF-8') if row[0][-1] != '+' else row[0][:-1]
                          for row in result]
            else:
                result = [row[0] if row[0][-1] != '+' else row[0][:-1] for row in result]

            # delete duplicates and sort, then join with ';' delimiter
            return ';'.join(sorted(list(set(result)))) + ';'
        else:
            if self.transducer.analyze(token):
                result = self.transducer.displayVector

                if not result:
                    return None
                # delete '+' sign in the end of analysis

                result = [row[0] if row[0][-1] != u'+' else row[0][:-1] for row in result]
                # delete duplicates and sort, then join with ';' delimiter
                return ';'.join(sorted(list(set(result)))) + ';'
        return None

    def analyse(self, token):
        """
            Tag word
        """
        if sys.version_info < (3, 0) and not isinstance(token, unicode):
            token = token.decode('UTF-8')

        token_lower = token.lower()
        if token_lower in self.tokens:
            result = self.tokens[token_lower]
        elif token in [u'\n', u'\n\r']:
            result = 'NL'
        elif token.isdigit():
            result = 'Num'
        elif re.match(u'^[a-zA-Z-]+$', token):
            result = 'Latin'
        elif re.match(u'^[^а-яА-ЯөүһңҗәҺҮӨҖҢӘЁё]$', token):
            result = 'Sign'
        elif (re.match(u'[а-яА-ЯөӨүҮһҺңҢҗҖәӘЁё]+$', token)) \
             or (re.match(u'[а-яА-ЯөӨүҮһҺңҢҗҖәӘЁё]+-[а-яА-ЯөӨүҮһҺңҢҗҖЁёәӘ]+$', token)):
            if token.count("-") > 1:
                result = 'Error'
            else:
                res = self.lookup(token)
                if not res:
                    res = self.lookup(token_lower)
                if not res and self.is_russian_word(token):
                    res = 'Rus'
                if not res:
                    result = 'NR'
                else:
                    result = res
        else:
            result = 'Error'

        return result

    def lemma(self, token):
        """
            Returns the lemma of the word
        """
        analysed = self.analyse(token)
        if analysed:
            return sorted(list(set([res.split(u'+')[0] for res in analysed.strip(';').split(';')])))

    def pos(self, token):
        """
            Returns the part-of-speech of the word
            !TODO
        """
        analysed = self.analyse(token)
        if analysed:
            return sorted(list(set([res.split(u'+')[1] for res in analysed.strip(';').split(';')])))

    def is_russian_word(self, word):
        """
            Parses word and check if word is russian
        """
        parses = self.rusmorphan.parse(word)
        for parse in parses:
            if 'UNKN' not in parse.tag and \
                not isinstance(parse.methods_stack[0][0], KnownSuffixAnalyzer.FakeDictionary) and\
                not isinstance(parse.methods_stack[1][0] if len(parse.methods_stack) > 1 else None,\
                                UnknownPrefixAnalyzer) and\
                isinstance(parse.methods_stack[0][0], DictionaryAnalyzer):
                return True
        return False

    def process_tokens(self, tokens):
        return dict([(token, self.analyse(token))
                     for token in set(tokens)
                     if token not in [u'\n\r', u'\r']])

    def analyse_text(self, text):
        """
            Process text and return as list of words
        """
        if sys.version_info < (3, 0) and not isinstance(text, unicode):
            text = text.decode('utf-8')

        text = fix(text)
        tokens = tokenize(text)
        tagged_tokens = self.process_tokens(tokens)

        sentence = []
        sentences = []

        for i, token in enumerate(tokens):
            tag = tagged_tokens.get(token, 'Error')
            if tag == 'Type1':
                sentence.append((token, tag))
                sentences.append(sentence)
                sentence = []
            elif tag == 'NL':
                if not self.params.get('ignore_newlines'):
                    sentence.append(('', 'NL'))
                    # if new line starts with uppercase, but prev line was not ended
                    if len(sentence) > 0 and i < len(tokens)-1 and tokens[i+1].istitle():
                        sentences.append(sentence)
                        sentence = []
            else:
                sentence.append((token, tag))

        if len(sentence) > 0:
            sentences.append(sentence)

        return len(tokens), len(tagged_tokens), len(sentences), sentences

    def process_text(self, text):
        """
            Takes plain text, tags and returns as plain text
        """
        if sys.version_info < (3, 0) and not isinstance(text, unicode):
            text = text.decode('utf-8')

        sdelimiter = self.params.get('sdelimiter', '\t')
        fdelimiter = self.params.get('fdelimiter', '\n')
        tokens_count, unique_tokens_count, sentences_count, sentences = self.analyse_text(text)

        result = []
        for sentence in sentences:
            for (token, tag) in sentence:
                result.append('%s%s%s' % (token, sdelimiter, tag))
        return fdelimiter.join(result)

    def disambiguate(self, text, disam_rules=None):
        """
            Parses text, analyses it and disambiguate morphological ambiguities
        """
        if not disam_rules:
            disam_rules = self.disam_rules
        tokens_count, unique_tokens_count, sentences_count, sentences = self.analyse_text(text)
        disamed_words_count = 0
        for sentence in sentences:
            for index, word in enumerate(sentence):
                if len(word[1].split(';')) >= 3:
                    # ambiguos word
                    result = disambiguate_word(sentence, index, disam_rules)
                    if result:
                        disamed_words_count += 1
                        sentence[index] = (word[0], result)
        return tokens_count, unique_tokens_count, sentences_count, disamed_words_count, sentences

    def disambiguate_text(self, text, disam_rules=None):
        """
            For Backward compatibility with older verions
        """
        tokens_count, unique_tokens_count, sentences_count, \
        disamed_words_count, sentences = self.disambiguate(text)
        return sentences