import glob
import os
from musiclang.lang import MusicLang
from musiclang.lang.regexes import remove_voices

class DBMusicLang:

    def __init__(self, database_path):

        self.database_path = database_path
        if not os.path.exists(self.database_path):
            os.makedirs(self.database_path)

    def midifiles(self, source):
        return glob.glob(os.path.join(source, '**/*.mid'), recursive=True)

    def save_to_database(self, source, views):
        import json

        filepath = os.path.join(self.database_path, source.split('/')[-1].replace('.mid', '.msl'))
        with open(filepath, 'w') as f:
            json.dump(views, f)

    def clean_text(self, text):
        return ' '.join(text.split())

    def get_views_from_source(self, source):
        score = MusicLang(source)
        score_not_factorized = MusicLang(source, factorize=False)
        score_half_factorized = MusicLang(source, factorize_full=False)
        chords = remove_voices(str(score))
        res = {
               'chords': self.clean_text(chords),
               'factorized': self.clean_text(str(score)),
               'raw': self.clean_text(str(score_not_factorized)),
               'half_factorized': self.clean_text(str(score_half_factorized))
               }

        return res

    def ingest_one(self, source):
        views = None
        try:
            views = self.get_views_from_source(source)
        except Exception as e:
            print(e)
            print(f'Error with {source}')
        if views is not None:
            self.save_to_database(source, views)

    def ingest_directory(self, source):
        for file in self.midifiles(source):
            self.ingest_one(file)


import datetime

from django.contrib.auth.models import User
from django.db import connection
from django.views.generic.list_detail import object_list
from django.views.generic.simple import direct_to_template

from emtools.emauth.decorators import require_admin
from emtools.emadmin.models import LogRecord, Schedule

from emtools import utils

OLD_DAYS = 7

@require_admin
def view_log(request):
    return object_list(request, LogRecord.objects.all(),
                       paginate_by=23,
                       template_name='emadmin/log.html',
                       extra_context={'tab': 'log'},
                       template_object_name='log')

@require_admin
def view_status(request):
    return direct_to_template(request, 'emadmin/status.html',
                              extra_context={'tab': 'status',
                                             'schedule_list':
                                                 Schedule.objects.all()})

IGNORED_GROUPS = ['Registered', 'Awaiting Activation', 'Banned']

@require_admin
def view_groups(request):
    db = utils.connect('emforum')
    c = db.cursor()
    c.execute("SELECT gid, title FROM mybb_usergroups")
    gid2name = dict(c.fetchall())
    c.execute("SELECT uid, username, lastactive, usergroup, additionalgroups "
              "FROM mybb_users")
    group_users = {}
    now = datetime.datetime.utcnow()
    for (mybb_uid, mybb_username, lastactive,
         usergroup, additionalgroups) in c.fetchall():
        try:
            user = User.objects.get(profile__mybb_uid=mybb_uid)
        except User.DoesNotExist:
            user = None
        lastactive = datetime.datetime.utcfromtimestamp(lastactive)
        delta = now - lastactive
        gids = [usergroup] + [int(gid) for gid in additionalgroups.split(",")
                              if gid != '']
        for gid in gids:
            group = gid2name.get(gid, "<Group %i>" % gid)
            if group in IGNORED_GROUPS:
                continue
            group_users.setdefault(group, [])
            group_users[group].append({'name': mybb_username,
                                       'user': user,
                                       'lastactive': lastactive,
                                       'days': delta.days,
                                       'cssclass': ('veryold'
                                                    if delta.days > OLD_DAYS
                                                    else '')})
    for users in group_users.values():
        users.sort(lambda a, b: cmp(a['name'].lower(), b['name'].lower()))
    groups = [(name, len(users), users)
              for (name, users) in group_users.items()]
    groups.sort(lambda a, b: cmp(a[0].lower(), b[0].lower()))
    return direct_to_template(request, 'emadmin/groups.html',
                              extra_context={'tab': 'groups',
                                             'group_list': groups})

@require_admin
def view_stats(request):
    conn = utils.connect('emmisc')
    c = conn.cursor()
    c.execute("SELECT table_schema, "
              "       SUM(table_rows) AS rows, "
              "       SUM(data_length + index_length) AS size "
              "FROM information_schema.TABLES "
              "GROUP BY table_schema "
              "ORDER BY size DESC")
    mysql_databases = c.fetchall()
    c.execute("SELECT table_name, "
              "       table_rows AS rows, "
              "       data_length + index_length AS size "
              "FROM information_schema.TABLES "
              "ORDER BY size DESC")
    mysql_tables = c.fetchall()
    c = connection.cursor()
    c.execute("SELECT 'eve', "
              "       (SELECT SUM(n_live_tup) FROM pg_stat_all_tables), "
              "       pg_database_size('eve')")
    postgresql_databases = c.fetchall()
    c.execute("""
SELECT CASE WHEN nspname = 'public'
            THEN relname
            ELSE nspname || '.' || relname
       END AS "relation",
       C.reltuples::numeric AS rows,
       pg_total_relation_size(C.oid) AS "total_size"
  FROM pg_class C
  LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
  WHERE nspname NOT IN ('pg_catalog', 'information_schema')
    AND C.relkind <> 'i'
    AND nspname !~ '^pg_toast'
  ORDER BY pg_total_relation_size(C.oid) DESC
""")
    postgresql_tables = c.fetchall()
    return direct_to_template(request, 'emadmin/stats.html',
                              extra_context={'tab': 'stats',
                                             'mysql_databases': mysql_databases,
                                             'mysql_tables': mysql_tables,
                                             'postgresql_databases': postgresql_databases,
                                             'postgresql_tables': postgresql_tables})


from pyspark import SparkContext
from pyspark.streaming import StreamingContext
import json
import requests

def sentimentValue(sentiment):
  if (sentiment == "positive"):
    return(1)
  elif (sentiment == "negative"):
    return(-1)
  else:
    return(0)

def getLineSentiment(line):
  words = line.lower().split()
  tsent = 0
  for word in words:
    wsent = 0
    try:
      wsent = sentimentValue(dwsent[word.replace("#","").replace("@","")].encode('ascii'))
    except:
      wsent = 0
    finally:
      tsent = tsent + wsent
  if tsent > 0:
    return("positive")
  elif tsent < 0:
    return("negative")
  else:
    return("neutral")

def getShow(line):
  words = line.lower().split()
  tsent = 0
  for word in words:
    try:
      show = shows[word.replace("#","").replace("@","").replace(":","")].encode('ascii')
      return(show)
    except:
      None

def getText(line):
  j = json.loads(line);
  return((j["created_at"], j["id"], j["text"]))

def postToSolr(t, rdd):
  solr_url = "http://auisbigdatabox.cloudapp.net:8983/solr/project/update?commitWithin=5000"
  d = json.dumps(rdd.map(lambda x: x).collect())
  print(d)
  h = {'content-type': 'application/json'}
  r = requests.post(solr_url, data=d, headers=h)
  print(r.status_code)
  #r2 = requests.get("http://auisbigdatabox.cloudapp.net:8983/solr/admin/cores?action=RELOAD&core=project_shard1_replica1")
  #print(r2.status_code)

sc = SparkContext()

dwsent = dict(sc.textFile("/data/dictionary.tsv").map(lambda line: line.split("\t")) \
  .map(lambda line: (line[2], line[5])).collect())
shows = dict(sc.textFile("/data/shows.txt").map(lambda line: (line.lower().encode('ascii'), line)).collect())

ssc = StreamingContext(sc, 1)
ds = ssc.textFileStream("/project/rawdata")
ods = ds.window(30, 30).map(lambda line: getText(line)) \
  .map(lambda line: {"created_dt": line[0], "show_s": getShow(line[2]), "sentiment_s": getLineSentiment(line[2]), "text_t": line[2]})
ods.foreachRDD(postToSolr)

ssc.start()
ssc.awaitTermination()


from typing import List, Dict, Any
import time
from stv.models.asynchronous.global_state import GlobalState
from stv.models.asynchronous.local_model import LocalModel
from stv.models.asynchronous.local_transition import LocalTransition, SharedTransition
from stv.models import SimpleModel

class GlobalModel:
    """
    Represents global model.

    :param local_models:
    :param reduction:

    :ivar _model:
    :ivar _local_models:
    :ivar _reduction:
    :ivar _states:
    :ivar _transitions:
    :ivar _dependent:
    :ivar _pre:
    :ivar _agents_count:
    :ivar _stack1:
    :ivar _stack2:
    :ivar _G:
    :ivar coalition:
    :ivar _stack1_dict:
    :ivar _transitions_count
    """

    def __init__(self, local_models: List[LocalModel], reduction: List[str]):
        self._model: SimpleModel = None
        self._local_models: List[LocalModel] = local_models
        self._reduction: List[str] = reduction
        self._states: List[GlobalState] = []
        self._transitions: List = []
        self._dependent: List[List[List[int]]] = []
        self._pre: List[List[List[LocalTransition]]] = []
        self._agents_count: int = 0
        self._states_dict: Dict[str, int] = dict()
        self._stack1: List[Any] = []
        self._stack2: List[int] = []
        self._G: List = []
        self.coalition: List = []
        self._stack1_dict: Dict[str, int] = dict()
        self._transitions_count: int = 0

    @property
    def model(self):
        """The model."""
        return self._model

    @property
    def states_count(self):
        return len(self._states)

    @property
    def transitions_count(self):
        return self._transitions_count

    def generate(self, reduction: bool = False):
        """
        Generates model.
        :param reduction: Should reductions be used.
        :return: None.
        """
        self._agents_count = len(self._local_models)
        self._model = SimpleModel(self._agents_count)
        self._add_to_stack(GlobalState.initial_state(self._agents_count))
        self._add_index_to_transitions()
        self._compute_dependent_transitions()
        self._compute_shared_transitions()
        if reduction:
            self._iter_por()
        else:
            self._compute()

    def _add_index_to_transitions(self):
        for agent_id in range(self._agents_count):
            for i in range(len(self._local_models[agent_id].transitions)):
                for j in range(len(self._local_models[agent_id].transitions[i])):
                    self._local_models[agent_id].transitions[i][j].i = i
                    self._local_models[agent_id].transitions[i][j].j = j

    def _compute_shared_transitions(self):
        replace = []
        for agent_id in range(self._agents_count):
            for i in range(len(self._local_models[agent_id].transitions)):
                for j in range(len(self._local_models[agent_id].transitions[i])):
                    transition = self._local_models[agent_id].transitions[i][j]
                    if not transition.shared:
                        continue

                    shared_transition = self._create_shared_transition(transition, agent_id)
                    shared_transition.transition_list.sort(key=lambda tran: tran.agent_id)
                    replace.append((agent_id, i, j, shared_transition))

        for rep in replace:
            agent_id, i, j, shared_transition = rep
            self._local_models[agent_id].transitions[i][j] = shared_transition

    def _create_shared_transition(self, transition: LocalTransition, agent_id: int) -> SharedTransition:
        shared_transition = SharedTransition(transition)
        for agent_id2 in range(self._agents_count):
            if agent_id == agent_id2:
                continue

            if self._local_models[agent_id2].has_action(transition.action):
                for transition2 in self._local_models[agent_id2].get_transitions():
                    if transition2.action == transition.action:
                        shared_transition.add_transition(transition2)
                        break

        return shared_transition

    def _available_transitions_in_state_for_agent(self, state: GlobalState, agent_id: int) -> List[LocalTransition]:
        """
        Computes a list of transitions available for the specified agent in the given state.
        :param state: Global state.
        :param agent_id: Agent identifier.
        :return: List of local transitions.
        """
        agent_state_id: int = state.local_states[agent_id]
        all_transitions: List[LocalTransition] = self._local_models[agent_id].private_transitions_from_state(
            agent_state_id)
        all_transitions += self._local_models[agent_id].shared_transitions_from_state(agent_state_id)
        return list(filter(lambda transition: transition.check_conditions(state), all_transitions))

    def _enabled_transitions_in_state(self, state: GlobalState) -> List[List[LocalTransition]]:
        """
        Computes all enabled transitions for the given global state.
        :param state:
        :return:
        """
        all_transitions = []
        for agent_id in range(len(self._local_models)):
            all_transitions.append(self._available_transitions_in_state_for_agent(state, agent_id))

        result = []
        for agent_id in range(self._agents_count):
            result.append(self._enabled_transitions_for_agent(agent_id, all_transitions))

        return result

    def _enabled_transitions_for_agent(self, agent_id: int, all_transitions: List[List[LocalTransition]]):
        """
        Computes enabled transitions for given agent based on the transitions from the global state.
        :param agent_id: Agent identifier.
        :param all_transitions: List containing all of the transitions going out from specific global state.
        :return:
        """
        result = []
        for transition in all_transitions[agent_id]:
            if not transition.shared:
                result.append(transition)
                continue

            if self._check_if_shared_transition_is_enabled(transition, agent_id, all_transitions):
                result.append(transition)

        return result

    def _check_if_shared_transition_is_enabled(self, transition: LocalTransition, agent_id: int,
                                               all_transitions: List[List[LocalTransition]]) -> bool:
        is_ok = True
        for agent_id2 in range(len(self._local_models)):
            if agent_id2 == agent_id:
                continue

            if self._local_models[agent_id2].has_action(transition.action):
                is_ok = False
                for transition2 in all_transitions[agent_id2]:
                    if transition2.shared and transition2.action == transition.action:
                        is_ok = True
                        break

            if not is_ok:
                return False

        return True

    def _enabled_transitions_in_state_single_item_set(self, state: GlobalState):
        enabled = self._enabled_transitions_in_state(state)
        result = set()
        for agent_id in range(self._agents_count):
            for transition in enabled[agent_id]:
                result.add(transition.to_tuple())
                if not transition.shared:
                    continue
                for agent_id2 in range(agent_id + 1, self._agents_count):
                    i = 0
                    for transition2 in enabled[agent_id2]:
                        if transition2.shared and transition2.action == transition.action:
                            enabled[agent_id2].pop(i)
                            break
                        i += 1
        return result

    def _new_state_after_private_transition(self, state: GlobalState, transition: LocalTransition):
        agent_id = transition.agent_id
        new_state = GlobalState.copy_state(state)
        new_state.set_local_state(agent_id, self._local_models[agent_id].get_state_id(transition.state_to))
        new_state.increment_counter(agent_id)
        new_state = self._copy_props_to_state(new_state, transition)
        return new_state

    def _new_state_after_shared_transition(self, state: GlobalState, actual_transition):
        new_state = GlobalState.copy_state(state)
        agents = []
        for act_tran in actual_transition:
            new_state.increment_counter(act_tran[0])
            new_state.set_local_state(act_tran[0], self._local_models[act_tran[0]].get_state_id(
                act_tran[1].state_to))
            new_state = self._copy_props_to_state(new_state, act_tran[1])
            agents.append(act_tran[0])
        return new_state, agents

    def _new_state_after_shared_transitions_list(self, state: GlobalState, transitions: List[LocalTransition]):
        new_state = GlobalState.copy_state(state)
        for transition in transitions:
            new_state.set_local_state(transition.agent_id,
                                      self._local_models[transition.agent_id].get_state_id(transition.state_to))
            new_state = self._copy_props_to_state(new_state, transition)
        return new_state

    def _compute_next_for_state(self, state: GlobalState, current_state_id: int):
        all_transitions = self._enabled_transitions_in_state(state)
        visited = []
        for agent_id in range(len(self._local_models)):
            self._compute_next_for_state_for_agent(state, current_state_id, agent_id, visited, all_transitions)

    def _compute_next_for_state_for_agent(self, state: GlobalState, current_state_id: int, agent_id: int, visited: [],
                                          all_transitions: []):
        for transition in all_transitions[agent_id]:
            if transition.shared and transition.action not in visited:
                visited.append(transition.action)
                actual_transition = [(agent_id, transition)]
                for n_a_id in range(agent_id + 1, len(self._local_models)):
                    for n_tr in all_transitions[n_a_id]:
                        if n_tr.shared and n_tr.action == transition.action:
                            actual_transition.append((n_a_id, n_tr))
                            break
                new_state, agents = self._new_state_after_shared_transition(state, actual_transition)
                new_state_id = self._add_state(new_state)
                self._add_transition(current_state_id, new_state_id, transition.action, agents)
            elif not transition.shared:
                new_state = self._new_state_after_private_transition(state, transition)
                new_state_id = self._add_state(new_state)
                self._add_transition(current_state_id, new_state_id, transition.action, [agent_id])

    def _copy_props_to_state(self, state: GlobalState, transition: LocalTransition) -> GlobalState:
        for prop in transition.props:
            if transition.props[prop] == "?":
                pass
            elif type(transition.props[prop]) is str:
                if transition.props[prop] in state.props:
                    state.set_prop(prop, state.props[transition.props[prop]])
            elif type(transition.props[prop]) is bool:
                if not transition.props[prop]:
                    state.remove_prop(prop)
                else:
                    state.set_prop(prop, transition.props[prop])
            else:
                state.set_prop(prop, transition.props[prop])
        return state

    def _state_find(self, state: GlobalState):
        if state.to_str() in self._states_dict:
            return self._states_dict[state.to_str()]

        return -1

    def _compute_dependent_transitions(self):
        for agent_id in range(self._agents_count):
            self._dependent.append([])
            agent_transitions = self._local_models[agent_id].get_transitions()
            for i in range(0, len(agent_transitions)):
                self._dependent[agent_id].append([agent_id])
                for agent2_id in range(self._agents_count):
                    if agent_id == agent2_id:
                        continue

                    if self._local_models[agent2_id].has_action(agent_transitions[i].action):
                        self._dependent[agent_id][i].append(agent2_id)

    def _is_in_G(self, state: GlobalState):
        for st in self._G:
            if st.equal(state):
                return True
        return False

    def _find_state_on_stack1(self, state: GlobalState):
        str_state = state.to_str()

        if str_state in self._stack1_dict:
            return self._stack1_dict[str_state]

        return -1

    def _add_to_stack(self, state: GlobalState):
        str_state = state.to_str()

        if str_state in self._stack1_dict:
            return False
        else:
            self._stack1.append(state)
            self._stack1_dict[state.to_str()] = len(self._stack1) - 1
            return True

    def _pop_from_stack(self):
        self._stack1_dict[self._stack1[-1].to_str()] = -1
        self._stack1.pop()

    def _dfs_por(self):
        """
        Recursive partial order reductions algorithm.
        :return: None.
        """
        g = self._stack1[-1]
        reexplore = False

        i = self._find_state_on_stack1(g)

        if i != -1 and i != len(self._stack1) - 1:
            if not self._stack2:
                depth = 0
            else:
                depth = self._stack2[-1]
            if i > depth:
                reexplore = True
            else:
                self._pop_from_stack()
                return

        if not reexplore and self._is_in_G(g):
            self._pop_from_stack()
        g_state_id = self._add_state(g)
        en_g = self._enabled_transitions_in_state_single_item_set(g)
        if len(en_g) > 0:
            if not reexplore:
                E_g = self._ample(g)
            if len(E_g) == 0:
                E_g = en_g
            if E_g == en_g:
                self._stack2.append(len(self._stack1))
            for tup in E_g:
                a = self._local_models[tup[0]].transitions[tup[1]][tup[2]]
                g_p = self._successor(g, a)
                g_p_state_id = self._add_state(g_p)
                self._add_transition(g_state_id, g_p_state_id, a.action, [a.agent_id])  # TODO agents ids
                if self._add_to_stack(g_p):
                    self._dfs_por()
        if len(self._stack2) == 0:
            depth = 0
        else:
            depth = self._stack2[-1]
        if depth == len(self._stack1):
            self._stack2.pop()
        self._pop_from_stack()

    def _iter_por(self):
        """
        Iterative partial order reduction algorithm.
        :return: None.
        """
        dfs_stack = [1]
        while len(dfs_stack) > 0:
            dfs = dfs_stack.pop()
            if dfs == 1:
                g = self._stack1[-1]
                reexplore = False
                i = self._find_state_on_stack1(g)
                if i != -1 and i != len(self._stack1) - 1:
                    if len(self._stack2) == 0:
                        depth = 0
                    else:
                        depth = self._stack2[-1]
                    if i > depth:
                        reexplore = True
                    else:
                        self._pop_from_stack()
                        return

                if not reexplore and self._is_in_G(g):
                    self._pop_from_stack()
                    return
                self._G.append(g)
                g_state_id = self._add_state(g)
                E_g = []
                en_g = self._enabled_transitions_in_state_single_item_set(g)
                if len(en_g) > 0:
                    if not reexplore:
                        E_g = self._ample(g)
                    if len(E_g) == 0:
                        E_g = en_g
                    if E_g == en_g:
                        self._stack2.append(len(self._stack1))
                    dfs_stack.append(-1)
                    for tup in E_g:
                        a = self._local_models[tup[0]].transitions[tup[1]][tup[2]]
                        g_p = self._successor(g, a)
                        g_p_state_id = self._add_state(g_p)
                        self._add_transition(g_state_id, g_p_state_id, a.action, [a.agent_id])
                        if self._add_to_stack(g_p):
                            dfs_stack.append(1)
            elif dfs == -1:
                if len(self._stack2) == 0:
                    depth = 0
                else:
                    depth = self._stack2[-1]
                if depth == len(self._stack1):
                    self._stack2.pop()
                self._pop_from_stack()

    def _ample(self, state: GlobalState):
        """
        Computes ample set for given state.
        :param state: Global state.
        :return: Ample set.
        """
        V = self._enabled_transitions_in_state_single_item_set(state)
        while V:
            alpha = V.pop()
            V.add(alpha)
            X = {alpha}
            U = {alpha}
            DIS = set()
            while X and not X.difference(V):
                DIS.update(self._enabled_for_x(X))
                X = self._dependent_for_x(X, DIS, U)
                U.update(X)
            if not X and not self._check_for_cycle(state, U):
                return U
            V.difference_update(U)
        return {}

    def _check_for_cycle(self, state: GlobalState, X: set):
        for tup in X:
            transition = self._local_models[tup[0]].transitions[tup[1]][tup[2]]
            successor_state = self._successor(state, transition)
            if self._find_state_on_stack1(successor_state) != -1:
                return True
        return False

    def _enabled_for_x(self, X):
        result = set()

        for tup in X:
            transition = self._local_models[tup[0]].transitions[tup[1]][tup[2]]
            if isinstance(transition, SharedTransition):
                for transition2 in transition.transition_list:
                    for tr in self._local_models[transition2.agent_id].get_transitions():
                        if tr.state_from != transition2.state_from:
                            result.add(tr.to_tuple())
            else:
                for tr in self._local_models[transition.agent_id].get_transitions():
                    if tr.state_from != transition.state_from:
                        result.add(tr.to_tuple())

        return result

    def _dependent_for_x(self, X, DIS, U):  # !!!!
        result = set()
        for tup in X:
            transition = self._local_models[tup[0]].transitions[tup[1]][tup[2]]
            if isinstance(transition, SharedTransition):
                for transition2 in transition.transition_list:
                    for tr in self._local_models[transition2.agent_id].get_transitions():
                        if tr.to_tuple() not in DIS and tr.to_tuple() not in U:
                            result.add(tr.to_tuple())
            else:
                for tr in self._local_models[transition.agent_id].get_transitions():
                    if tr.to_tuple() not in DIS and tr.to_tuple() not in U:
                        result.add(tr.to_tuple())

        return result

    def _successor(self, state: GlobalState, transition: LocalTransition):
        if not isinstance(transition, SharedTransition):
            return self._new_state_after_private_transition(state, transition)
        else:
            return self._new_state_after_shared_transitions_list(state, transition.transition_list)

    def _add_state(self, state: GlobalState):
        state_id = self._state_find(state)
        if state_id == -1:
            state_id = len(self._states)
            state.id = state_id
            self._states.append(state)
            self._states_dict[state.to_str()] = state_id

        state.id = state_id
        return state_id

    def _add_transition(self, state_from: int, state_to: int, action: str, agents: List[int]):
        while len(self._transitions) <= state_from:
            self._transitions.append([])

        self._transitions[state_from].append({'from': state_from, 'to': state_to, 'action': action, 'agents': agents})
        self._transitions_count += 1
        self._model.add_transition(state_from, state_to, self._create_list_of_actions(action, agents))

    def _create_list_of_actions(self, action: str, agents: List[int]):
        actions = ['' for _ in range(self._agents_count)]
        for agent_id in agents:
            actions[agent_id] = action
        return actions

    def _compute(self):
        """
        Compute global model.
        :return:
        """
        state = GlobalState.initial_state(len(self._local_models))
        self._states.append(state)
        i = 0
        while i < len(self._states):
            state = self._states[i]
            current_state_id = i
            i += 1

            self._compute_next_for_state(state, current_state_id)

    def agent_name_to_id(self, agent_name: str) -> int:
        for agent_id in range(len(self._local_models)):
            if self._local_models[agent_id].agent_name == agent_name:
                return agent_id
        return -1

    def agent_name_coalition_to_ids(self, agent_names: List[str]) -> List[int]:
        agent_ids = []
        for agent_name in agent_names:
            agent_ids.append(self.agent_name_to_id(agent_name))
        return agent_ids

    def walk(self):
        print("Simulation start")
        current_state_id = 0
        while True:
            print("Current state:")
            self._states[current_state_id].print()
            print("Transitions:")
            for i in range(0, len(self._transitions[current_state_id])):
                print(f"{i}: {self._transitions[current_state_id][i]}")

            id = int(input("Select transition: "))
            current_state_id = self._transitions[current_state_id][id]['to']

    def print_dependent_transitions(self):
        for agent_id in range(self._agents_count):
            print(f"Agent {self._local_models[agent_id].agent_name}")
            agent_transitions = self._local_models[agent_id].get_transitions()
            for i in range(0, len(agent_transitions)):
                print("Transition:")
                agent_transitions[i].print()
                print("Dependent:")
                for agent2_id in self._dependent[agent_id][i]:
                    print(f"{self._local_models[agent2_id].agent_name}")
                print()
            print()

    def print(self):
        for model in self._local_models:
            model.print()

    def set_coalition(self, coalition: List[str]):
        self.coalition = self.agent_name_coalition_to_ids(coalition)

if __name__ == "__main__":
    from stv.models.asynchronous.parser import GlobalModelParser

    file_name = "voting.txt"
    model = GlobalModelParser().parse(file_name)
    # model.parse("train_controller.txt")
    # model.parse("selene.txt")
    # model.print()
    coalition = ["Coercer1"]
    model.set_coalition(coalition)
    print(f"Coalition: {coalition}")
    start = time.process_time()
    model.generate(reduction=True)
    end = time.process_time()
    print()
    print(f"Model generated in {end - start} seconds.")
    print(f"Model has {model.states_count} states.")
    print(f"Model has {model.transitions_count} transitions.")
    print()
    # model.walk()


import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer

'''
cd appendices/ && python makeindex.py && cd ../
'''

import re,glob

from nltk.corpus import stopwords
stop = list(stopwords.words('english'))
stop.extend('section subsection subsubsection figure table item paragraph begin end caption autoref includegraph  png pdf textwidth endsubfigure subfigure the is this centering label textbf textit emph fig cite citep width height includegraphics sec use ce ch footnote figures shown use used much often within show contain'.split())

files = glob.glob('../*_*.tex')
files.sort()

corpus = []
title = []
for f in files:
    try:
        dir = open(f,'r').read()
        loc = re.findall(r'\\subimport\{([^\}]+)',dir)[0]
        text = open('../%scombigned.tex'%loc,'r').read().lower()
        title.append(re.findall(r'\\chapter\{([^\}]+)',dir)[0])

        text = re.sub(r'[\._\W\s\d]+',' ',re.sub(r'-|([^\\]\%[^\n]+)','',text ))
        corpus.append(text)
    except:None

vectorizer = TfidfVectorizer(analyzer = 'word',stop_words=set(stop))
vectors = vectorizer.fit_transform(corpus)
names = vectorizer.get_feature_names()
data = vectors.todense().tolist()
# Create a dataframe with the results
df = pd.DataFrame(data, columns=names,index=title)

tfidf = '''
\\chapter{Chapter Keywords}
This section uses the Term Frequency Inverse Document Frequency to determine the keywords of each chapter - a techneque which has been described in \\autoref{ch3} and \\cite{frankenstein}. Text size corresponds to the importance of each word.

'''

N = 100;
for i in df.iterrows():
    print('\n\n',i[0])
    tfidf += '\n\n \\section{%s}\n {\\fontfamily{cmtt}\\selectfont \\parbox{\\textwidth}{\n'%i[0]
    rankings = i[1].sort_values(ascending=False)[:N]
    print(rankings)

    for i in rankings.iteritems():
        tfidf += '\\size{%d}{ %s }'%(1+i[1]*100,i[0].upper())

    #tfidf=tfidf[:-2]
    tfidf+='} }'

with open('keywords.tex','w') as f:
    f.write(tfidf)


from django.shortcuts import render
from django.views.generic import CreateView, DeleteView, ListView, UpdateView, DetailView
from lugati.lugati_points_of_sale.models import SalesPoint, City
from lugati.lugati_points_of_sale.forms import CityForm
from django.contrib.sites.models import get_current_site
import math
from django.http import HttpResponse
import json
from django.views.decorators.csrf import csrf_exempt
from django.core.urlresolvers import resolve

import logging
logger = logging.getLogger(__name__)

def get_points_of_sale_control_panel(request):
    return render(request, 'lugati_admin/points_of_sale/points_of_sale_control_panel.html')

def get_points_of_sale_tree(request):
    dt = []
    sales_points = SalesPoint.objects.all()
    for sales_point in sales_points:

        cur_node = {
            'label': sales_point.name,
            'uid': sales_point.id,
            'data': {
                'definition': sales_point.name
            }
        }

        dt.append(cur_node)

    return HttpResponse(json.dumps(dt))

class UpdateSalesPoint(UpdateView):
    model = SalesPoint
    #template_name = 'lugati_admin/points_of_sale/create_sales_point.html'
    template_name = 'lugati_admin/points_of_sale/update_sales_point_new.html'
    success_url = '/lugati_admin/points_of_sale/'

#cities
def get_cities_tree_ajax(request):
    cur_site = get_current_site(request)

    def get_cities_tree(region=None, level=0):
        dt = []
        cities = City.objects.filter(site=cur_site).filter(region=region).order_by('is_region', 'name')
        for city in cities:
            cur_node = {
                'label': city.name,
                'uid': city.id,
                'data': {
                    'definition': city.name
                }
            }
            children = City.objects.filter(region = city)
            if children.exists():
                cur_node['children'] = get_cities_tree(region=city, level=level+1)
            dt.append(cur_node)
        return dt

    res_dt = get_cities_tree()
    return HttpResponse(json.dumps(res_dt))

def list_cities(request):
    return render(request, 'lugati_admin/cities/list_cities_new.html')

class CreateCity(CreateView):
    model = City
    template_name = 'lugati_admin/cities/create_city.html'
    success_url = '/lugati_admin/points_of_sale/cities/'

    form_class = CityForm

    def get_initial(self):
        res = {}

        match = resolve(self.request.path)
        if match.url_name == 'create_region':
            res['is_region'] = True
        else:
            res['is_region'] = False
        cur_site = get_current_site(self.request)
        res['site'] = cur_site
        return res
    def form_invalid(self, form):
        if self.request.is_ajax():
            form_errors = {}
            for k in form.errors:
                form_errors[k] = form.errors[k][0]
            res_dt = {
                'error': True,
                'form_errors': form_errors
            }
            return HttpResponse(json.dumps(res_dt))
    def form_valid(self, form):
        self.object = form.save()

        if self.request.is_ajax():
            res_dt = {
                'error': False
            }
            return HttpResponse(json.dumps(res_dt))

class UpdateCity(UpdateView):
    model = City
    template_name = 'lugati_admin/cities/update_city.html'
    success_url = '/lugati_admin/points_of_sale/cities/'

    def form_invalid(self, form):
        if self.request.is_ajax():
            res_dt = {
                'error': True
            }
            return HttpResponse(json.dumps(res_dt))
    def form_valid(self, form):
        self.object = form.save()

        if self.request.is_ajax():
            res_dt = {
                'error': False
            }
            return HttpResponse(json.dumps(res_dt))

class DeleteCity(DeleteView):
    model = City
#~


import click
import rubrik_oracle_module as rbk

@click.command()
@click.argument('host_cluster_db')
@click.option('--force', '-f', is_flag=True, help='Force unmount')
def cli(host_cluster_db, force):
    """
    This will unmount a Rubrik live mount using the database name and the live mount host.

\b
    Args:
        host_cluster_db (str): The hostname the database is running on : The database name
        force (bool): Force the unmount
\b
    Returns:
        unmount_info (dict): Status of the unmount request.
    """
    rubrik = rbk.connect_rubrik()
    cluster_info = rbk.get_cluster_info(rubrik)
    timezone = cluster_info['timezone']['timezone']
    print("Connected to cluster: {}, version: {}, Timezone: {}.".format(cluster_info['name'], cluster_info['version'], timezone))
    host_cluster_db = host_cluster_db.split(":")
    live_mount_ids = rbk.get_oracle_live_mount_id(rubrik, cluster_info['id'], host_cluster_db[1], host_cluster_db[0])
    if not live_mount_ids:
        raise RubrikOracleUnmountError("No live mounts found for {} live mounted on {}. ".format(host_cluster_db[1], host_cluster_db[0]))
    else:
        unmount_info = []
        for live_mount_id in live_mount_ids:
            unmount_response = rbk.live_mount_delete(rubrik, live_mount_id, force)
            print("Live mount id: {} Unmount status: {}.".format(live_mount_id, unmount_response['status']))
            unmount_info.append(unmount_response)
        return unmount_info

class RubrikOracleUnmountError(rbk.NoTraceBackWithLineNumber):
    """
        Renames object so error is named with calling script
    """
    pass

if __name__ == "__main__":
    cli()


#coding:utf-8
from django.shortcuts import render_to_response, RequestContext
from main.models import Blog
from django.http import Http404
from django.views.decorators.cache import cache_page
from django.core.paginator import Paginator
from django.core.paginator import EmptyPage
from django.core.paginator import PageNotAnInteger
# Create your views here.

@cache_page(86400)
def blog_list(request):
    limit = 6
    topics = Blog.objects.all()
    paginator = Paginator(topics, limit)

    page = request.GET.get('page')
    try:
        topics = paginator.page(page)
    except PageNotAnInteger:
        topics = paginator.page(1)
    except EmptyPage:
        topics = paginator.page(paginator.num_pages)
    return render_to_response("blog_list.html", {"topics": topics})

@cache_page(604800)
def blog_show(request, id=''):
    try:
        blog = Blog.objects.get(id=id)
    except Blog.DoesNotExist:
        raise Http404
    return render_to_response("blog_show.html", {"blog": blog}, context_instance=RequestContext(request))


from os import path

import numpy as np
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score

def load_data():
    '''
    SMSSpamCollection.txt数据集
    第一列是短信的label
    ham：非垃圾短信
    spam：垃圾短信
    \t键后面是短信的正文
    '''
    file_name = path.dirname(__file__) + "/SMSSpamCollection.txt"
    X, y = [], []
    with open(file_name,'r', encoding='UTF-8') as file:
        line = file.readline()
        while line:
            d = line.split("\t")
            X.append(d[1])  #短信正文
            y.append(d[0])  #label
            line = file.readline()

##    print(X[:10])
##    print(y[:10])

    return X, y

def about_train_test_split():
    '''
    X_train, X_test, y_train, y_test =sklearn.model_selection.train_test_split(train_data,train_target,
    test_size=0.4, random_state=0,stratify=y_train)
    参数说明：
    # train_data：所要划分的样本特征集
    # train_target：所要划分的样本结果
    # test_size：样本占比，如果是整数的话就是样本的数量
    # random_state：是随机数的种子。
    # stratify是为了保持split前类的分布。比如有100个数据，80个属于A类，20个属于B类。
    如果train_test_split(... test_size=0.25, stratify = y_all), 那么split之后数据如下：
    training: 75个数据，其中60个属于A类，15个属于B类。
    testing: 25个数据，其中20个属于A类，5个属于B类。

    用了stratify参数，training集和testing集的类的比例是 A：B= 4：1，等同于split前的比例（80：20）。
    通常在这种类分布不平衡的情况下会用到stratify。
    将stratify=X就是按照X中的比例分配
    将stratify=y就是按照y中的比例分配
    '''
    # step 1 调用load_data方法读入数据
    X, y = load_data()
    a = {'ham':0,'spam':1}
    y=[a[s]for s in y]

    # step 2 利用缺省参数调用train_test_split将数据划分为训练集和测试集(X_train, X_test, y_train, y_test)
    X_train_raw, X_test_raw, y_train, y_test=train_test_split(X,y)

    # step 3 创建TfidfVectorizer对象vectorizer, 用训练集的短信文本训练vectorizer，
    #        利用训练好的vectorizer抽取训练集的短信文本和测试集的短信文本的tfidf
    vectorizer=TfidfVectorizer()
    X_train = vectorizer.fit_transform(X_train_raw)
    X_test = vectorizer.transform(X_test_raw)
    print(vectorizer.get_feature_names())
    print("\n抽取训练集的短信文本",X_train)
    print("\n抽取测试集的短信文本", X_test)

    # step 4 创建LogisticRegression对象classifier, 用训练集训练classifier
    #        利用训练好的classifier对测试集进行预测
    #        输出前10个测试样本的预测值、实际值和短信文本
    #        输出分类准确率（利用accuracy_score函数）
    #        返回混淆矩阵（利用confusion_matrix函数）
    classifier = LogisticRegression(solver = 'lbfgs')
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    for i, prediction in enumerate(y_pred[:10]):
        print('Prediction: %s. Real: %s. Message: %s' % (prediction, y_test[i], X_test_raw[i]))

    print('Accuracy:', accuracy_score(y_test, y_pred))

    return confusion_matrix(y_test, y_pred)

def show_confusion_matrix(c_mat):
    print(c_mat)
    plt.matshow(c_mat)
    plt.title('Confusion matrix')
    plt.colorbar()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.show()

def about_cross_val_score():
    '''
    cross_val_score(estimator, X, y=None, groups=None,
                    scoring=None, cv=’warn’, n_jobs=None, verbose=0, fit_params=None,
                    pre_dispatch=‘2*n_jobs’, error_score=’raise-deprecating’)
    参数：
    estimator： 需要使用交叉验证的算法
    X： 输入样本数据
    y： 样本标签
    groups： 将数据集分割为训练/测试集时使用的样本的组标签（一般用不到）
    scoring： 交叉验证最重要的就是他的验证方式，选择不同的评价方法，会产生不同的评价结果。
    cv： 交叉验证折数或可迭代的次数
    n_jobs： 同时工作的cpu个数（-1代表全部）
    verbose： 详细程度
    fit_params： 传递给估计器（验证算法）的拟合方法的参数
    pre_dispatch： 控制并行执行期间调度的作业数量。
    error_score： 如果在估计器拟合中发生错误，要分配给该分数的值（一般不需要指定）
    '''
    X, y = load_data()
    a = {'ham':0,'spam':1}
    y = [ a[s] for s in y]  #将label转换为0和1
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(X)
    classifier = LogisticRegression(solver = 'lbfgs')

    # step 1 利用5折交叉验证对classifier进行训练，输出accuracy
    scores = cross_val_score(classifier, X, y, cv = 5, scoring = 'accuracy')
    print('Accuracy:', np.mean(scores), scores)

    # step 2 利用5折交叉验证对classifier进行训练，输出precision
    precision = cross_val_score(classifier, X, y, cv=5, scoring='precision')
    print('Precision:', np.mean(precision), precision)

    # step 3 利用5折交叉验证对classifier进行训练，输出recall
    recall = cross_val_score(classifier, X, y, cv=5, scoring='recall')
    print('Recall:', np.mean(recall), recall)

    # step 4 利用5折交叉验证对classifier进行训练，输出f1
    f1 = cross_val_score(classifier, X, y, cv=5, scoring='f1')
    print('F1:', np.mean(f1), f1)

from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score

def about_P_R_curve():
    '''
    sklearn.metrics.precision_recall_curve(y_true, probas_pred, *, pos_label=None, sample_weight=None)
    Compute precision-recall pairs for different probability thresholds

    Note: this implementation is restricted to the binary classification task.
    The precision is the ratio tp / (tp + fp) where tp is the number of true positives and fp the number of false positives. The precision is intuitively the ability of the classifier not to label as positive a sample that is negative.
    The recall is the ratio tp / (tp + fn) where tp is the number of true positives and fn the number of false negatives. The recall is intuitively the ability of the classifier to find all the positive samples.
    The last precision and recall values are 1. and 0. respectively and do not have a corresponding threshold. This ensures that the graph starts on the y axis.

    Parameters
    y_true array, shape = [n_samples]
    True binary labels. If labels are not either {-1, 1} or {0, 1}, then pos_label should be explicitly given.

    probas_pred array, shape = [n_samples]
    Estimated probabilities or decision function.

    pos_label int or str, default=None
    The label of the positive class. When pos_label=None, if y_true is in {-1, 1} or {0, 1}, pos_label is set to 1, otherwise an error will be raised.

    sample_weight array-like of shape (n_samples,), default=None
    Sample weights.

    Returns
    precision array, shape = [n_thresholds + 1]
    Precision values such that element i is the precision of predictions with score >= thresholds[i] and the last element is 1.

    recall array, shape = [n_thresholds + 1]
    Decreasing recall values such that element i is the recall of predictions with score >= thresholds[i] and the last element is 0.

    thresholds array, shape = [n_thresholds <= len(np.unique(probas_pred))]
    Increasing thresholds on the decision function used to compute precision and recall.
    '''
    X, y = load_data()
    a = {'ham':0,'spam':1}
    y = [ a[s] for s in y]  #将label转换为0和1
    X_train_raw, X_test_raw, y_train, y_test = train_test_split(X, y)

    vectorizer = TfidfVectorizer()
    X_train = vectorizer.fit_transform(X_train_raw)
    X_test = vectorizer.transform(X_test_raw)

    classifier = LogisticRegression(solver = 'lbfgs')
    classifier.fit(X_train, y_train)

    # step 1 调用predict_proba方法得到测试集的预测概率
    y_pred = classifier.predict_proba(X_test)

    average_precision = average_precision_score(y_test, y_pred[:,1])

    # step 2 调用precision_recall_curve计算P-R曲线上各点的precision和recall
    precision,recall,thresholds=precision_recall_curve(y_test,y_pred[:,1])

    # step 3 调用matplotlib.pyplot的plot方法画P-R曲线
    plt.plot(recall, precision)

    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.ylim([0.0, 1.05])
    plt.xlim([0.0, 1.0])
    plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))
    plt.show()

from sklearn.metrics import roc_curve, auc
def about_ROC_curve():
    '''
    sklearn.metrics.roc_curve(y_true, y_score, pos_label=None, sample_weight=None, drop_intermediate=True)

    主要参数：
    y_true：真实的样本标签，默认为{0，1}或者{-1，1}。如果要设置为其它值，则 pos_label 参数要设置为特定值。
            例如要令样本标签为{1，2}，其中2表示正样本，则pos_label=2
    y_score：对每个样本的预测结果。
    pos_label：正样本的标签

    返回值：
    fpr：False positive rate
    tpr：True positive rate (Recall)
    thresholds
    '''
    X, y = load_data()
    a = {'ham':0,'spam':1}
    y = [ a[s] for s in y]

    X_train_raw, X_test_raw, y_train, y_test = train_test_split(X, y)

    # step 1 创建TfidfVectorizer对象vectorizer, 用训练集的短信文本训练vectorizer，
    #        利用训练好的vectorizer抽取训练集的短信文本和测试集的短信文本的tfidf
    vectorizer=TfidfVectorizer()
    X_train = vectorizer.fit_transform(X_train_raw)
    X_test = vectorizer.transform(X_test_raw)
    print(vectorizer.get_feature_names())
    print("\n抽取训练集的短信文本",X_train)
    print("\n抽取测试集的短信文本", X_test)

    # step 2 创建LogisticRegression对象classifier, 用训练集训练classifier
    #        利用训练好的classifier对测试集进行预测,得到预测概率值
    classifier = LogisticRegression(solver = 'lbfgs')
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict_proba(X_test)

    # step 3 根据真实标签和预测概率值调用roc_curve计算false_positive_rate、recall和thresholds
    #        调用auc计算ROC曲线下方的面积大小roc_auc
    false_positive_rate,recall,thresholds=roc_curve(y_test,y_pred[:,1])
    roc_auc=auc(false_positive_rate,recall)
    # step 4 画ROC曲线
    plt.plot(false_positive_rate, recall, 'b', label='AUC = %0.2f' %roc_auc)

    plt.title('Receiver Operating Characteristic')
    plt.legend(loc='lower right')
    plt.plot([0, 1], [0, 1], 'r--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.0])
    plt.ylabel('Recall')
    plt.xlabel('Fall-out')
    plt.show()

if __name__=="__main__":
    c_m = about_train_test_split()
    show_confusion_matrix(c_m)
    about_cross_val_score()
    about_P_R_curve()
    about_ROC_curve()


from django.conf import settings
from django.conf.urls.static import static
from django.conf.urls import url
from django.urls import path
from pictureURL.views import *
from . import views

urlpatterns = [
    path("", dashboard, name="dashboard"),
    path('login/', login_page , name="login_page"),
    path('logout/', logout, name='logout'),
    path('AllCampaigns/', campaign_list, name="campaign_list"),
    path('campaigndetail/<slug:auid>', campaigndetail, name='campaigndetail'),
    path("NewCampaign/", Upload_Campaign.as_view(),  name="new_campaign"),
    path("edit/<int:id>/", EditListView.as_view(), name="edit_campaign"),
    path("Delete_Campaign/<int:pk>/", delete_campaign, name="delete_campaign"),
    path("smsform/", Publish.as_view(),  name="textform"),
    path("result/", search, name="search"),
    # path("Calender/", calender, name="calender")

]
if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)


# Standard library imports
import os
import json
import requests

# Open json file with stored links
with open('stored_links.json', 'r') as data_file:
    data = json.load(data_file)
    for list_with_offers in data:
        for item in list_with_offers:
            print(item)
            offers = os.listdir('offers')
            number_of_offers = len(offers)
            i = number_of_offers + 1
            url = item
            response = requests.get(url)
            content = response.text
            if i<10:
                path = f'offers/offer_000{i}.html'
            elif i<100:
                path = f'offers/offer_00{i}.html'
            elif i<1000:
                path = f'offers/offer_0{i}.html'
            else:
                path = f'offers/offer_{i}.html'
            # Get offers data and save each offer data as separate html file
            with open(path, 'w', encoding='utf-8') as output_data:
                output_data.write(content)


import numpy as np
import pdb
import mdtraj
import simtk.unit as u
import bilayer_analysis_functions
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

import plot_ay
plot_ay.setDefaults()

def find_blocks(number):
    max_power = int(np.log10(number))
    all_blocks = [[int(10**num), int(2.5*10**num), int(5*10**num), int(7.5*10**num)]
                        for num in np.arange(0, max_power+1, dtype=int)]
    all_blocks = np.array(all_blocks).flatten()
    all_blocks = np.array([val for val in all_blocks if val <= number])
    return all_blocks

kb = u.BOLTZMANN_CONSTANT_kB
#kb = u.BOLTZMANN_CONSTANT_kB * u.AVOGADRO_CONSTANT_NA
T = 305 * u.kelvin
#traj = mdtraj.load('trajectory.dcd', top='npt.gro')
areas = []
n_frames = 0
for chunk in mdtraj.iterload('trajectory.dcd', top='npt.gro'):
    area_chunk = chunk.unitcell_lengths[:,0] * chunk.unitcell_lengths[:,1]
    areas = np.concatenate((areas, area_chunk))
    n_frames += chunk.n_frames
areas *= u.nanometer**2
avg_area = np.mean(areas)
mean_sq_fluc = np.sum((areas - avg_area)**2)/len(areas)
ka = kb*T*avg_area/mean_sq_fluc
print(ka.in_units_of(u.dyne/u.centimeter))
print(ka.in_units_of(u.newton/u.meter))

#for i, frame in enumerate(traj):
#    frame.time = 10*i

# we have 10,000 frames for 100ns
# we now have 50,000 frames for 500ns
# Let's declare block size
block_sizes = find_blocks(n_frames)
#block_sizes = [100,200,300,400, 500,600,700,800,900, 1000, 2000, 5000,
            #10000, 20000, 25000, n_frames]
fig, ax = plt.subplots(1,2)
for block_size in block_sizes:
    n_blocks = int(n_frames / block_size)
    all_ka = []
    for i in range(n_blocks):
        window_areas = areas[i : i + block_size]
        window_avg = np.mean(window_areas)
        window_fluc = np.sum((window_areas - window_avg)**2)/len(window_areas)
        ka = kb*T*window_avg/window_fluc
        all_ka.append(ka.in_units_of(u.dyne/u.centimeter)._value)
    mean = np.mean(all_ka)
    #se = np.std(all_ka)/np.sqrt(n_blocks)
    se = np.std(all_ka)
    ax[0].scatter([block_size], [se])
    ax[1].scatter([block_size], [mean])
    print("{} blocksize, {} blocks, {} mean, {} se".format(block_size, n_blocks,
                                    mean, se))
ax[0].set_xlabel("Block size (frames)")
ax[0].set_ylabel("Standard error")
ax[1].set_ylabel("Compressibilty Modulus (dyne/cm)")
ax[1].set_xlabel("Block size (frames)")
fig.tight_layout()
fig.savefig('block_comparison.png')
plt.close(fig)

# Look at autocorrelations
# 10000 frames for 100ns, we have
# 50000 frames for 500ns
corr_length = 25000
dt = 10
time_origins = np.arange(0, n_frames - corr_length, step=dt)
all_corrs = []
for origin in time_origins:
    avg_area = np.mean(areas[origin:origin+corr_length])
    area_corr = (((areas[origin]-avg_area)*(areas[origin: origin+corr_length]-avg_area)) /
                    ((areas[origin]-avg_area)**2))
    all_corrs.append(area_corr)
all_corrs = np.asarray(all_corrs)
time_vals = np.arange(0, corr_length)/100
fig, ax = plt.subplots(1,1)
ax.plot(time_vals, np.mean(all_corrs, axis=0))
fig.tight_layout()
fig.savefig('correlations.png')
plt.close(fig)

fig, ax = plt.subplots(1,1)
ax.plot(np.arange(0, len(areas)), areas)
ax.set_xlabel("Frames")
ax.set_ylabel("Box Area")
ax.grid()
fig.tight_layout()
fig.savefig('areas.png')
plt.close(fig)


# coding=utf-8
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TODO: Add a description here."""

from __future__ import absolute_import, division, print_function

import copy
import xml.etree.ElementTree as etree

import datasets

logger = datasets.logging.get_logger(__name__)

# Find for instance the citation on arxiv or on the dataset repo/website
_CITATION = """\
"""

# You can copy an official description
_DESCRIPTION = """\
NLM produces a baseline set of MEDLINE/PubMed citation records in XML format for download on an annual basis. The annual baseline is released in December of each year. Each day, NLM produces update files that include new, revised and deleted citations. See our documentation page for more information.
"""

_HOMEPAGE = "https://www.nlm.nih.gov/databases/download/pubmed_medline.html"

_LICENSE = ""

# TODO: Add link to the official dataset URLs here
# The HuggingFace dataset library don't host the datasets but only point to the original files
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
_URLs = [f"ftp://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed21n{i:04d}.xml.gz" for i in range(1, 1063)]

# Copyright Ferry Boender, released under the MIT license.
# Modified by @Narsil to handle more oddities
def deepupdate(target, src):
    """Deep update target dict with src
    For each k,v in src: if k doesn't exist in target, it is deep copied from
    src to target. Otherwise, if v is a list, target[k] is extended with
    src[k]. If v is a set, target[k] is updated with v, If v is a dict,
    recursively deep-update it.

    Examples:
    >>> t = {'name': 'Ferry', 'hobbies': ['programming', 'sci-fi']}
    >>> deepupdate(t, {'hobbies': ['gaming']})
    >>> print(t)
    {'name': 'Ferry', 'hobbies': ['programming', 'sci-fi', 'gaming']}
    """
    for k, v in src.items():
        if k in target and isinstance(target[k], int) and isinstance(v, str):
            try:
                v = int(v)
            except Exception:
                pass
        if k in target and type(target[k]) != type(v):
            logger.warning(f"Ignoring field {k} it's a {type(v)} and we expect a {type(target[k])}")
            continue

        if type(v) == list:
            if k not in target:
                target[k] = copy.deepcopy(v)
            elif isinstance(target[k], list):
                target[k].extend(v)
            elif isinstance(target[k], str):
                # Very special case to handle `AbstractText` which sometimes end up
                # being a list.
                new_v = " ".join(el for el in v if isinstance(el, str))
                target[k] = new_v
            else:
                logger.warning(f"Ignoring field {k} it's a {type(v)} and we expect a {type(target[k])}")
        elif type(v) == dict:
            if k not in target:
                target[k] = copy.deepcopy(v)
            elif isinstance(target[k], dict):
                deepupdate(target[k], v)
            else:
                logger.warning(f"Ignoring field {k} it's a {type(v)} and we expect a {type(target[k])}")
        elif type(v) == set:
            if k not in target:
                target[k] = v.copy()
            elif isinstance(target[k], set):
                target[k].update(v.copy())
            else:
                logger.warning(f"Ignoring field {k} it's a {type(v)} and we expect a {type(target[k])}")
        else:
            if isinstance(target[k], (list, tuple, dict)):
                logger.warning(f"Ignoring field {k} it's a {type(v)} and we expect a {type(target[k])}")
                continue

            target[k] = copy.copy(v)

def default_date():
    return {"Year": 0, "Month": 0, "Day": 0}

def default_inline_article():
    return {
        # 'Journal': Journal,
        "Abstract": {"AbstractText": ""},
        "ArticleTitle": "",
        # 'Pagination': {'MedlinePgn': datasets.Value('string')},
        "AuthorList": {"Author": []},
        "Language": "",
        "GrantList": {
            "Grant": [],
        },
        "PublicationTypeList": {"PublicationType": []},
    }

def default_article():
    return {
        "MedlineCitation": {
            "PMID": 0,
            "DateCompleted": default_date(),
            "NumberOfReferences": 0,
            "DateRevised": default_date(),
            "Article": default_inline_article(),
            "MedlineJournalInfo": {"Country": ""},
            "ChemicalList": {"Chemical": []},
            "CitationSubset": "",
            "MeshHeadingList": {"MeshHeading": []},
        },
        "PubmedData": {
            "ArticleIdList": [{"ArticleId": []}],
            "PublicationStatus": "",
            "History": {"PubMedPubDate": []},
            "ReferenceList": [],
        },
    }

class Pubmed(datasets.GeneratorBasedBuilder):
    """Pubmed citations records"""

    BUILDER_CONFIGS = [
        datasets.BuilderConfig(name="2021", description="The 2021 annual record", version=datasets.Version("1.0.0")),
    ]

    # FILLED automatically from features
    SIMPLE_KEYS = {"PubmedArticleSet"}
    LIST_KEYS = {"PubmedArticle"}
    IGNORE_KEYS = set()

    def fill_keys_from_features(self, features):
        if isinstance(features, dict):
            for key, value in features.items():
                if isinstance(value, datasets.Sequence):
                    self.LIST_KEYS.add(key)
                    self.fill_keys_from_features(value.feature)
                else:
                    self.SIMPLE_KEYS.add(key)
                    self.fill_keys_from_features(value)

    def xml_to_dictionnary(self, parentElement):
        data = {}
        if parentElement.tag in {"AbstractText", "ArticleTitle"}:
            # XXX
            # Very special case, it will contain html leading to having very odd structure
            tag = parentElement.tag
            string = etree.tostring(parentElement).decode("utf-8").strip()
            inner_string = string[len(f"<{tag}>") : -len(f"</{tag}>")]
            return {parentElement.tag: inner_string}

        for child in list(parentElement):
            child.text = child.text if (child.text is not None) else " "
            key = child.tag
            if len(child) == 0:
                value = child.text.strip()
            else:
                value = self.xml_to_dictionnary(child)
                if isinstance(value, dict) and set(value.keys()) == {key}:
                    value = value[key]

            if key in data:
                old_value = data[key]
                if isinstance(old_value, dict):
                    data[key] = [old_value, value]
                elif isinstance(old_value, list):
                    data[key].append(value)
            elif key in self.LIST_KEYS:
                data[key] = [value]
            elif key in self.SIMPLE_KEYS:
                data[key] = value
            elif key in self.IGNORE_KEYS:
                continue
            else:
                logger.info(f"Ignoring key {key} from {parentElement.tag}")
                self.IGNORE_KEYS.add(key)

        # Filling defaults
        if parentElement.tag == "MeshHeading" and "QualifierName" not in data:
            data["QualifierName"] = ""
        elif parentElement.tag == "Author":
            if "ForeName" not in data:
                data["ForeName"] = ""
            if "Initials" not in data:
                data["Initials"] = ""
            if "LastName" not in data:
                data["LastName"] = ""
            if "CollectiveName" not in data:
                data["CollectiveName"] = ""
        elif parentElement.tag == "JournalIssue":
            if "Volume" not in data:
                data["Volume"] = ""
            if "Issue" not in data:
                data["Issue"] = ""
        elif parentElement.tag == "Grant" and "GrantID" not in data:
            data["GrantID"] = ""

        return {parentElement.tag: data}

    def _info(self):
        Date = {
            "Year": datasets.Value("int32"),
            "Month": datasets.Value("int32"),
            "Day": datasets.Value("int32"),
        }

        MeshHeading = {"DescriptorName": datasets.Value("string"), "QualifierName": datasets.Value("string")}

        MedlineJournalInfo = {
            "Country": datasets.Value("string"),
            # Too inconsistent
            # 'MedlineTA': datasets.Value('string'),
            # 'NlmUniqueID': datasets.Value('string'),
            # 'ISSNLinking': datasets.Value('string'),
        }
        Chemical = {
            "RegistryNumber": datasets.Value("string"),
            "NameOfSubstance": datasets.Value("string"),
        }
        # Too inconsistent in the data to be used
        # Journal = {
        #         'ISSN': datasets.Value('string'),
        #         'JournalIssue': {
        #             'Volume': datasets.Value('string'),
        #             'Issue': datasets.Value('string'),
        #         },
        #         # 'PubDate': Date,
        #         'Title': datasets.Value('string'),
        #         'ISOAbbreviation': datasets.Value('string')
        #         }
        Author = {
            "LastName": datasets.Value("string"),
            "ForeName": datasets.Value("string"),
            "Initials": datasets.Value("string"),
            "CollectiveName": datasets.Value("string"),
        }
        Reference = {
            "Citation": datasets.Value("string"),
            "CitationId": datasets.Value("int32"),
        }
        Grant = {
            "GrantID": datasets.Value("string"),
            "Agency": datasets.Value("string"),
            "Country": datasets.Value("string"),
        }
        Article = {
            # 'Journal': Journal,
            "Abstract": {"AbstractText": datasets.Value("string")},
            "ArticleTitle": datasets.Value("string"),
            # Too inconistent
            # 'Pagination': {'MedlinePgn': datasets.Value('string')},
            "AuthorList": {"Author": datasets.Sequence(Author)},
            "Language": datasets.Value("string"),
            "GrantList": {
                "Grant": datasets.Sequence(Grant),
            },
            "PublicationTypeList": {"PublicationType": datasets.Sequence(datasets.Value("string"))},
        }
        features = datasets.Features(
            {
                "MedlineCitation": {
                    "PMID": datasets.Value("int32"),
                    "DateCompleted": Date,
                    "NumberOfReferences": datasets.Value("int32"),
                    "DateRevised": Date,
                    "Article": Article,
                    "MedlineJournalInfo": MedlineJournalInfo,
                    "ChemicalList": {"Chemical": datasets.Sequence(Chemical)},
                    "CitationSubset": datasets.Value("string"),
                    "MeshHeadingList": {
                        "MeshHeading": datasets.Sequence(MeshHeading),
                    },
                },
                "PubmedData": {
                    "ArticleIdList": datasets.Sequence({"ArticleId": datasets.Sequence(datasets.Value("string"))}),
                    "PublicationStatus": datasets.Value("string"),
                    "History": {"PubMedPubDate": datasets.Sequence(Date)},
                    "ReferenceList": datasets.Sequence(Reference),
                },
            }
        )
        self.fill_keys_from_features(features)
        return datasets.DatasetInfo(
            # This is the description that will appear on the datasets page.
            description=_DESCRIPTION,
            # This defines the different columns of the dataset and their types
            features=features,  # Here we define them above because they are different between the two configurations
            # If there's a common (input, target) tuple from the features,
            # specify them here. They'll be used if as_supervised=True in
            # builder.as_dataset.
            supervised_keys=None,
            # Homepage of the dataset for documentation
            homepage=_HOMEPAGE,
            # License for the dataset if available
            license=_LICENSE,
            # Citation for the dataset
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager):
        """Returns SplitGenerators."""
        dl_dir = dl_manager.download_and_extract(_URLs)
        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN,
                # These kwargs will be passed to _generate_examples
                gen_kwargs={"filenames": dl_dir},
            ),
        ]

    def update_citation(self, article):
        """
        ArticleId and ArticleIdList are already used field name so we rewrite and
        flatten those as {Citation, CitationId}.
        """
        citations = []
        try:
            list_ = article["PubmedData"]["ReferenceList"]
        except Exception:
            return

        for ref in list_:
            if "Reference" not in ref:
                continue
            for re in ref["Reference"]:
                if "Citation" not in re:
                    continue
                citation = re["Citation"]
                if "ArticleIdList" not in re:
                    continue
                for r in re["ArticleIdList"]:
                    if "ArticleId" not in r:
                        continue
                    for rr in r["ArticleId"]:
                        try:
                            citation = {"Citation": citation, "CitationId": int(rr)}
                        except Exception:
                            continue
                        citations.append(citation)
        article["PubmedData"]["ReferenceList"] = citations

    def _generate_examples(self, filenames):
        """ Yields examples. """
        id_ = 0
        for filename in filenames:
            try:
                tree = etree.parse(filename)
                root = tree.getroot()
                xmldict = self.xml_to_dictionnary(root)
            except etree.ParseError:
                logger.warning(f"Ignoring file {filename}, it is malformed")
                continue

            for article in xmldict["PubmedArticleSet"]["PubmedArticle"]:
                self.update_citation(article)
                new_article = default_article()

                try:
                    deepupdate(new_article, article)
                except Exception:
                    logger.warning(f"Ignoring article {article}, it is malformed")
                    continue

                try:
                    _ = self.info.features.encode_example(new_article)
                except Exception as e:
                    logger.warning(f"Ignore example because {e}")
                    continue
                yield id_, new_article
                id_ += 1


# -*- coding: utf-8 -*-

"""This module populates the tables of bio2bel_wikipathways."""

import logging
import sys
from typing import Iterable, List, Mapping, Optional, Union

import click
from tqdm import tqdm

import bio2bel_hgnc
from bio2bel.manager.bel_manager import BELManagerMixin
from bio2bel.manager.flask_manager import FlaskMixin
from bio2bel.manager.namespace_manager import BELNamespaceManagerMixin
from compath_utils import CompathManager
from pybel.constants import BIOPROCESS, NAME, PROTEIN
from pybel.manager.models import Namespace, NamespaceEntry
from pybel.struct.graph import BELGraph
from .constants import MODULE_NAME, WIKIPATHWAYS
from .models import Base, Pathway, Protein, protein_pathway
from .parser import parse_gmt_file

__all__ = [
    'Manager',
]

log = logging.getLogger(__name__)
logging.getLogger("urllib3").setLevel(logging.WARNING)

class Manager(CompathManager, BELNamespaceManagerMixin, BELManagerMixin, FlaskMixin):
    """Protein-pathway memberships."""

    module_name = MODULE_NAME
    _base = Base
    edge_model = protein_pathway
    flask_admin_models = [Pathway, Protein]
    namespace_model = pathway_model = Pathway
    pathway_model_identifier_column = Pathway.wikipathways_id
    protein_model = Protein

    identifiers_recommended = 'WikiPathways'
    identifiers_pattern = r'WP\d{1,5}(\_r\d+)?$'
    identifiers_miriam = 'MIR:00000076'
    identifiers_namespace = 'wikipathways'
    identifiers_url = 'http://identifiers.org/wikipathways/'

    """Override views in _make_admin"""

    def _add_admin(self, app, **kwargs):
        from flask_admin import Admin
        from flask_admin.contrib.sqla import ModelView

        class PathwayView(ModelView):
            """Pathway view in Flask-admin."""

            column_searchable_list = (
                Pathway.wikipathways_id,
                Pathway.name
            )

        class ProteinView(ModelView):
            """Protein view in Flask-admin."""

            column_searchable_list = (
                Protein.entrez_id,
                Protein.hgnc_symbol,
                Protein.hgnc_id
            )

        admin = Admin(app, **kwargs)
        admin.add_view(PathwayView(Pathway, self.session))
        admin.add_view(ProteinView(Protein, self.session))
        return admin

    def summarize(self) -> Mapping[str, int]:
        """Summarize the database."""
        return dict(
            pathways=self._count_model(Pathway),
            proteins=self._count_model(Protein),
        )

    def query_pathway_by_name(self, query: str, limit: Optional[int] = None) -> List[Pathway]:
        """Return all pathways having the query in their names.

        :param query: query string
        :param limit: limit result query
        """
        q = self.session.query(Pathway).filter(Pathway.name.contains(query))

        if limit:
            q = q.limit(limit)

        return q.all()

    def get_or_create_pathway(
            self,
            wikipathways_id: str,
            name: Optional[str] = None,
            species: Optional[str] = None,
    ) -> Pathway:
        """Get an pathway from the database or creates it.

        :param wikipathways_id: WikiPathways identifier
        :param name: name of the pathway
        :param species: species in which the pathway was described
        """
        pathway = self.get_pathway_by_id(wikipathways_id)

        if pathway is None:
            pathway = Pathway(
                wikipathways_id=wikipathways_id,
                name=name,
                species=species,
            )
            self.session.add(pathway)

        return pathway

    def get_or_create_protein(self, entrez_id: str, hgnc_symbol: str, hgnc_id: str) -> Protein:
        """Get an pathway from the database or creates it.

        :param entrez_id: entrez identifier
        :param hgnc_symbol: hgnc symbol
        :param hgnc_id: hgnc identifier
        """
        protein = self.get_protein_by_entrez_id(entrez_id)

        if protein is None:
            protein = Protein(
                entrez_id=entrez_id,
                hgnc_symbol=hgnc_symbol,
                hgnc_id=hgnc_id
            )

            self.session.add(protein)

        return protein

    def get_protein_by_entrez_id(self, entrez_id: str) -> Optional[Protein]:
        """Get a protein by its Entrez gene identifier."""
        return self.session.query(Protein).filter(Protein.entrez_id == entrez_id).one_or_none()

    def get_protein_by_hgnc_id(self, hgnc_id: str) -> Optional[Protein]:
        """Get a protein by its HGNC identifier."""
        return self.session.query(Protein).filter(Protein.hgnc_id == hgnc_id).one_or_none()

    def get_protein_by_hgnc_symbol(self, hgnc_symbol: str) -> Optional[Protein]:
        """Get a protein by its HGNC gene symbol."""
        return self.session.query(Protein).filter(Protein.hgnc_symbol == hgnc_symbol).one_or_none()

    """Methods to populate the DB"""

    def populate(self, url: Union[None, str, Iterable[str]] = None):
        """Populate the database.

        :param url: url from a GMT file
        """
        hgnc_manager = bio2bel_hgnc.Manager(engine=self.engine, session=self.session)
        if not hgnc_manager.is_populated():
            hgnc_manager.populate()

        if url is None or isinstance(url, str):
            pathways = parse_gmt_file(url=url)
        elif isinstance(url, Iterable):
            pathways = [
                pathway
                for u in url
                for pathway in parse_gmt_file(url=u)
            ]
        else:
            raise TypeError(f'Invalid type for url: {type(url)} ({url})')

        # Dictionaries to map across identifiers
        entrez_to_hgnc_symbol = hgnc_manager.build_entrez_id_symbol_mapping()
        hgnc_symbol_id = hgnc_manager.build_hgnc_symbol_id_mapping()

        entrez_id_protein = {}
        missing_entrez_ids = set()

        it = tqdm(pathways, desc='Loading WikiPathways')
        for pathway_name, species, wikipathways_id, gene_set in it:
            pathway = self.get_or_create_pathway(
                wikipathways_id=wikipathways_id,
                name=pathway_name.strip(),
                species=species,
            )

            for entrez_id in gene_set:
                if entrez_id in entrez_id_protein:
                    protein = entrez_id_protein[entrez_id]

                else:
                    hgnc_symbol = entrez_to_hgnc_symbol.get(entrez_id)

                    if not hgnc_symbol:
                        it.write(f"({species}) ncbigene:{entrez_id} has no HGNC symbol")
                        missing_entrez_ids.add(entrez_id)
                        continue

                    protein = self.get_or_create_protein(entrez_id, hgnc_symbol, hgnc_symbol_id[hgnc_symbol])
                    entrez_id_protein[entrez_id] = protein

                if pathway not in protein.pathways:
                    protein.pathways.append(pathway)

            self.session.commit()

        if missing_entrez_ids:
            log.warning("Total of {} missing ENTREZ".format(len(missing_entrez_ids)))

    @classmethod
    def _cli_add_populate(cls, main: click.Group) -> click.Group:
        @main.command()
        @click.option('--reset', is_flag=True, help='Nuke database first')
        @click.option('--force', is_flag=True, help='Force overwrite if already populated')
        @click.option('-u', '--url', multiple=True, help='URL of WikiPathways GMT files')
        @click.pass_obj
        def populate(manager: Manager, reset, force, url):
            """Populate the database."""
            if reset:
                click.echo('Deleting the previous instance of the database')
                manager.drop_all()
                click.echo('Creating new models')
                manager.create_all()

            if manager.is_populated() and not force:
                click.echo('Database already populated. Use --force to overwrite')
                sys.exit(0)

            manager.populate(url=url)

        return main

    """Methods to enrich BEL graphs"""

    def to_bel(self) -> BELGraph:
        """Return a new graph corresponding to the pathway.

        Example Usage:

        >>> manager = Manager()
        >>> manager.get_pathway_graph_by_id('WP61') # Notch signaling pathway
        """
        graph = BELGraph(
            name='WikiPathways Associations',
            version='1.0.0',
        )

        wikipathways_namespace = self.upload_bel_namespace()
        graph.namespace_url[wikipathways_namespace.keyword] = wikipathways_namespace.url

        hgnc_manager = bio2bel_hgnc.Manager(engine=self.engine, session=self.session)
        hgnc_namespace = hgnc_manager.upload_bel_namespace()
        graph.namespace_url[hgnc_namespace.keyword] = hgnc_namespace.url

        for pathway in tqdm(self.get_all_pathways(), total=self._count_model(Pathway)):
            for protein in pathway.proteins:
                pathway_bel = pathway.serialize_to_pathway_node()
                protein_bel = protein.serialize_to_protein_node()
                graph.add_part_of(protein_bel, pathway_bel)

        return graph

    def get_pathway_graph_by_id(self, wikipathways_id: str) -> Optional[BELGraph]:
        """Return a new graph corresponding to the pathway.

        :param wikipathways_id: WikiPathways identifier
        :return: A BEL Graph corresponding to the WikiPathways identifier

        Example Usage:

        >>> manager = Manager()
        >>> manager.get_pathway_graph_by_id('WP61') # Notch signaling pathway
        """
        pathway = self.get_pathway_by_id(wikipathways_id)

        if pathway is None:
            return

        graph = BELGraph(
            name=f'{pathway.name} ({pathway.species})',
            version='1.0.0',
        )

        wikipathways_namespace = self.upload_bel_namespace()
        graph.namespace_url[wikipathways_namespace.keyword] = wikipathways_namespace.url

        hgnc_manager = bio2bel_hgnc.Manager(engine=self.engine, session=self.session)
        hgnc_namespace = hgnc_manager.upload_bel_namespace()
        graph.namespace_url[hgnc_namespace.keyword] = hgnc_namespace.url

        pathway_node = pathway.serialize_to_pathway_node()

        for protein in pathway.proteins:
            graph.add_part_of(protein.serialize_to_protein_node(), pathway_node)

        return graph

    def enrich_wikipathways_pathway(self, graph: BELGraph) -> None:
        """Enrich all proteins belonging to WikiPathways pathway nodes in the graph."""
        for node in list(graph):
            if node.function == BIOPROCESS and node.namespace == WIKIPATHWAYS and NAME in node:
                pathway = self.get_pathway_by_name(node.name)
                for protein in pathway.proteins:
                    graph.add_part_of(protein.serialize_to_protein_node(), node)

    def enrich_wikipathways_protein(self, graph: BELGraph) -> None:
        """Enrich all WikiPathways pathways associated with proteins in the graph."""
        for node in list(graph):
            if node.function == PROTEIN and 'namespace' in node and node.namespace == 'HGNC':
                protein = self.get_protein_by_hgnc_symbol(node.name)
                for pathway in protein.pathways:
                    graph.add_part_of(node, pathway.serialize_to_pathway_node())

    def _create_namespace_entry_from_model(self, model: Pathway, namespace: Namespace) -> NamespaceEntry:
        """Create a namespace entry from the model."""
        return NamespaceEntry(encoding='B', name=model.name, identifier=model.wikipathways_id, namespace=namespace)

    @staticmethod
    def _get_identifier(model: Pathway) -> str:
        """Extract the identifier from a pathway mode."""
        return model.wikipathways_id


import os
import re

from django.shortcuts import render, redirect, get_object_or_404
from django.http import HttpResponse
from django.db import transaction
from django.core.exceptions import ValidationError
from django.contrib.auth.decorators import login_required
from django.conf import settings

from subdicts import utils

from dormitory_application_form.models import Applicant, DegreeProgram, Bracket, Attachment, ApplicationMeta
from dormitory_application_form.forms import ApplicantForm
from dormitory_application_form import listeners

def index(request):
    if request.user and request.user.is_authenticated():
        return redirect('application:applicants')

    application_ended = ApplicationMeta.objects.get(pk='application_ended')
    if application_ended.value == 'true':
        return render(request, 'dormitory_application_form/applicants.html', {'applicants': Applicant.objects.accepted()})

    context = {
        'applicant': Applicant(),
        'degree_programs': DegreeProgram.objects.all(),
        'brackets': Bracket.objects.all(),
        'meta': {
            'semester': ApplicationMeta.objects.get(pk='semester'),
            'school_year': ApplicationMeta.objects.get(pk='school_year')
        }
    }

    if request.method == 'POST':
        post = utils.parse(request.POST.copy())
        post['applicant']['father_deceased'] = 'father_deceased' in post['applicant'] and post['applicant']['father_deceased'] == 'on'
        post['applicant']['mother_deceased'] = 'mother_deceased' in post['applicant'] and post['applicant']['mother_deceased'] == 'on'
        if not post['applicant']['scholarship_allowance']:
            del post['applicant']['scholarship_allowance']

        applicant = ApplicantForm(post['applicant'])
        if applicant.is_valid():
            with transaction.atomic():
                applicant = applicant.save()
                for key in request.FILES:
                    applicant.attachment_set.create(
                        filename = request.FILES[key],
                        attachment_type = request.POST[key]
                    )
                applicant.email_application_sent()
            return redirect('application:done')
        else:
            post['applicant']['course'] = DegreeProgram.objects.get(pk=post['applicant']['course'])
            post['applicant']['tuition_discount'] = Bracket.objects.get(pk=post['applicant']['tuition_discount'])
            context['applicant'] = Applicant(**post['applicant'])
            context['errors'] = applicant.errors

    return render(request, 'dormitory_application_form/index.html', context)

def done(request):
    return render(request, 'dormitory_application_form/done.html')

@login_required
def applicants(request):
    applicants = Applicant.objects.order_by('tuition_discount__rank', 'annual_family_income', 'scholarship_allowance')
    return render(request, 'dormitory_application_form/applicants.html', {'applicants': applicants})

@login_required
def show(request, applicant_id):
    applicant = get_object_or_404(Applicant, pk=applicant_id)
    return render(request, 'dormitory_application_form/show.html', {'applicant': applicant})

@login_required
def delete(request, applicant_id):
    applicant = get_object_or_404(Applicant, pk=applicant_id)
    with transaction.atomic():
        applicant.delete()
        for attachment in applicant.attachment_set.all():
            attachment.delete()
    return redirect('application:applicants')

@login_required
def accept(request, applicant_id):
    applicant = get_object_or_404(Applicant, pk=applicant_id)
    applicant.accepted = True
    applicant.save()
    return redirect('application:applicants')

@login_required
def unaccept(request, applicant_id):
    applicant = get_object_or_404(Applicant, pk=applicant_id)
    applicant.accepted = False
    applicant.save()
    return redirect('application:applicants')

@login_required
def notify_accepted(request):
    for applicant in Applicant.objects.accepted():
        applicant.email_applicant_accepted()
    application_ended = ApplicationMeta.objects.get(pk='application_ended')
    application_ended.value = 'true'
    application_ended.save()
    return redirect('application:applicants')


import math
from sklearn.utils import resample

f = open("boostrap.csv", 'r',encoding="utf-8")
listA = f.readlines()
#print(len(listA))

row=[]
i=0
num_sample_row=0
for line in listA:
    if i==0:
        i=1
        continue
    # khi có continue thì sẽ bỏ qua các câu lệnh trong vòng lặp
    x = line.split(",")
    #x là dòng string
    num_sample_row=len(x)
    for i in range(0,len(x)):
        row.append(int(x[i]))
#print(row)
n_size = num_sample_row * 20 + 1;
list_train=[]

for i in range(0,n_size):
    new_row = resample(row, replace=True, n_samples=num_sample_row)
    list_train.append(new_row)
#print(list_train)

def mean(y):
    sum_mean=0
    for i in y:
        sum_mean+=i
    return sum_mean/len(y)

#Median: trung vị
def median(y):
    y.sort()
    if len(y) % 2 == 0:
        return (y[len(y)//2]+y[len(y)//2-1])/2
    else:
        return y[len(y)//2]

#IQR:
def IQR(y):
    y.sort()
    q50_index = len(y)//2
    q25_index = q50_index // 2
    q75_index = int(len(y)/4*3)
    return (y[q75_index] - y[q25_index])

def IQR_New(x):
    x.sort()
    Q3=np.percentile(x,75)
    Q1=np.percentile(x,25)
    return Q3-Q1

def std_deviation(y):
    #độ lệch chuẩn
    sum_std=0;
    for i in range(0,len(y)):
        sum_std+=pow((y[i]-mean(y)),2);
    return math.ceil(math.sqrt(sum_std/(len(y)-1)));
def Mean_distribution(listmean,numbin):
    for mean in listmean:
        for key in numbin:
            if (mean <= key):
                numbin[key]+=1
                break
    return numbin

list_mean = []
list_median = []
list_IQR = []
list_std_deviation = []
#print(list_train[0])
for line in list_train:
    list_mean.append(mean(line))
    list_median.append(median(line))
    list_IQR.append(IQR(line))
    list_std_deviation.append(std_deviation(line))
#print(len(list_mean))
print("")
print("--------------Confident interval of the mean Bootstrap-----------------")
total_sample = len(list_train) #tổng số mẫu : 201
print("Total sample: ",total_sample)
alpha = 0.05
print("alpha: ",alpha)
confident_level = 100*(1-2*alpha) #độ tin cậy
print("Confident level: ",confident_level)
# confidence interval : khoảng tin cậy
list_mean.sort()
lower_bound = list_mean[int(total_sample * alpha)-1] #vị trí của phần tử thứ 10 trong listmean
print("Lower bound: ",lower_bound)
upper_bound = list_mean[int(total_sample * (1-alpha))-1]
print("upper bound: ",upper_bound)

import pandas as pd
from prettytable import PrettyTable
# pd.options.display.max_columns = None
# pd.options.display.max_rows = None
print("--------------Bootstrap Statistic-----------------")
# intialise data of lists.
# data1 = {"Mean":list_mean,
#         "Median":list_median,
#         "Inter quartile range":list_IQR,
#         "Std deviation":list_std_deviation}
# # Create DataFrame
# df1 = pd.DataFrame(data1)
# print(df1)

t = PrettyTable(['STT','Mean', 'Median','IQR', 'Standard Division'])
for i in range(0, n_size):
    t.add_row([i+1,list_mean[i], list_median[i], list_IQR[i],list_std_deviation[i]])
print(t)

num_bin = {10: 0, 20: 0, 30: 0, 40: 0, 50: 0, 60:0, 70:0, 80:0, 90:0, 100:0}
num_bin=Mean_distribution(list_mean,num_bin)
list_bin = list(num_bin.keys())
list_frequency = list(num_bin.values())
list_probability = [round(i/total_sample,2) for i in list_frequency]
# số 2 là lấy 2 chữ số thập phân

# data2 = {"Bin":list_bin,
#         "Frequency":list_frequency,
#         "Probability":list_probability}
# df2 = pd.DataFrame(data2)

# new_row = ["SUM",total_sample,1]
# df2 = df2.append(pd.Series(new_row, index = df2.columns), ignore_index=True)
# print("")
# print("--------------Mean distribution-----------------")
# print(df2.to_string(index=False))

print("--------------Mean distribution-----------------")
df = PrettyTable(['Bin','Frequency', 'Probability'])
# df = pd.DataFrame(columns=['Bin','Frequency', 'Probability'])
i = 0
for key in num_bin:
    # df = df.append({'Bin': key,'Frequency':bin_number[key],'Probability':probability[i]}, ignore_index=True)
    df.add_row([key,num_bin[key], list_probability[i]])
    i += 1
df.add_row(["SUM",total_sample,1])
# df = df.append({'Bin': "SUM",'Frequency':total_sample,'Probability':1}, ignore_index=True)
print(df)
# print(df)

import matplotlib.pyplot as plt
import numpy as np
# x axis values
#tạo mảng từ 0->100 cách đều nhau 10
x = np.arange(0,100,10)
plt.plot(x, list_probability)
plt.xlabel('x - axis')
plt.ylabel('y - axis')
plt.title('My first graph!')
plt.bar(x, list_probability, color='purple', width = 5)  #vẽ khối trụ

# plt.axvline(lower_bound, linestyle = 'dotted',color='green',label='pyplot vertical line')
plt.axvline(lower_bound, color='green',label='pyplot vertical line')
plt.axvline(upper_bound, color='green',label='pyplot vertical line')
# function to show the plot
plt.show()


import re
import urllib2
from bs4 import BeautifulSoup

class RefWords(object):
    def __init__(self, word):
        content = urllib2.urlopen("http://dictionary.reference.com/browse/{0}".format(word), timeout=10).read()
        self.content = BeautifulSoup(content, "html.parser")

    def get_ipa(self):
        ipa = self.content.find('span', {'class': 'pron ipapron'})
        if ipa == None:
            return ""
        else:
            return ipa.string

    def get_def(self):
        def_str = u""
        soup = self.content.find('div', {'class': 'source-box'})
        for row in soup.find_all('section', {'class': 'def-pbk'}):
            text = row.get_text().replace('\n', '').replace('\r', '')
            count = 0
            for line in re.split('\d+\.(?!\d)', text):
                if line.strip() == "":
                    continue
                if count == 0:
                    def_str += line + '\n'
                else:
                    def_str += u"{0}. {1}\n".format(count, line)
                count += 1
            def_str += "\n"
        return def_str


# -*- coding: utf-8 -*-
##----------------------------------------------------------------------
## fm.reportoutages
##----------------------------------------------------------------------
## Copyright (C) 2007-2015 The NOC Project
## See LICENSE for details
##----------------------------------------------------------------------

## Python modules
import datetime
from collections import defaultdict
## Django modu;es
from django import forms
## NOC modules
from noc.fm.models.outage import Outage
from noc.sa.models.managedobject import ManagedObject
from noc.lib.app.simplereport import SimpleReport, TableColumn
from noc.lib.dateutils import total_seconds
from noc.lib.nosql import Q

class ReportForm(forms.Form):
    duration = forms.ChoiceField(choices=[
        (86400, "1 day"),
        (7 * 86400, "1 week"),
        (30 * 86400, "1 month")
    ])

class ReportOutagesApplication(SimpleReport):
    title = "Outages"

    form = ReportForm

    def get_data(self, duration, **kwargs):
        now = datetime.datetime.now()
        d = datetime.timedelta(seconds=int(duration))
        b = now - d
        outages = defaultdict(list)
        otime = defaultdict(int)
        q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)
        for o in Outage.objects.filter(q):
            start = max(o.start, b)
            stop = o.stop if o.stop else now
            outages[o.object] += [o]
            otime[o.object] = total_seconds(stop - start)
        td = total_seconds(d)
        # Load managed objects
        mos = list(otime)
        chunk = 500
        mo = {}
        while mos:
            for o in ManagedObject.objects.filter(id__in=mos[:chunk]):
                mo[o.id] = o
            mos = mos[chunk:]
        r = []
        for o in sorted(otime, key=lambda x: -otime[x]):
            m = mo.get(o)
            if not m:
                continue  # Hanging Outage
            dt = otime[o]
            downtime = "%02d:%02d:%02d" % (
                (dt // 3600) % 24,
                (dt // 60) % 60,
                dt % 60
            )
            if dt >= 86400:
                downtime = "%dd %s" % (dt // 86400, downtime)
            r += [(
                m.name,
                m.profile_name,
                m.platform,
                m.is_managed,
                m.get_status(),
                downtime,
                float(td - dt) * 100 / td,
                len(outages[o])
            )]

        return self.from_dataset(
            title=self.title,
            columns=[
                "Object", "Profile", "Platform",
                TableColumn("Managed", format="bool"),
                TableColumn("Status", format="bool"),
                TableColumn("Downtime", align="right"),
                TableColumn("Availability", align="right", format="percent"),
                TableColumn("Downs", align="right", format="integer")
            ],
            data=r,
            enumerate=True
        )


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from .viewsets import (
    CustomTextViewSet,
    DescriptionViewSet,
    HomePageViewSet,
    InventoryViewSet,
    LocationViewSet,
    LokationViewSet,
    NameViewSet,
    StoresViewSet,
    StorezViewSet,
)

from home.api.v1.viewsets import (
    SignupViewSet,
    LoginViewSet,
    HomePageViewSet,
    CustomTextViewSet,
)

router = DefaultRouter()
router.register("signup", SignupViewSet, basename="signup")
router.register("login", LoginViewSet, basename="login")
router.register("customtext", CustomTextViewSet)
router.register("homepage", HomePageViewSet)
router.register("location", LocationViewSet)
router.register("stores", StoresViewSet)
router.register("inventory", InventoryViewSet)
router.register("storez", StorezViewSet)
router.register("name", NameViewSet)
router.register("lokation", LokationViewSet)
router.register("description", DescriptionViewSet)

urlpatterns = [
    path("", include(router.urls)),
]