import pandas as pd
import numpy as np

from sklearn.model_selection import train_test_split
train = pd.read_csv("E://ML//Kaggle Titanic//train.csv")
test = pd.read_csv("E://ML//Kaggle Titanic//test.csv")
train = train.drop(['Name','Cabin','Ticket'],axis = 1)
test = test.drop(['Name','Cabin','Ticket'],axis = 1)
train["Age"].fillna( method ='ffill', inplace = True)
#print(train.isna().sum())
test["Age"].fillna( method ='ffill', inplace = True)
train.dropna()
test.dropna()

from sklearn.preprocessing import StandardScaler
replacement = {
    'S': 0,
    'Q': 1,
    'C': 2
}

train['Embarked'] = train['Embarked'].apply(lambda x: replacement.get(x))
train['Embarked'] = StandardScaler().fit_transform(train['Embarked'].values.reshape(-1, 1))
test['Embarked'] = test['Embarked'].apply(lambda x: replacement.get(x))
test['Embarked'] = StandardScaler().fit_transform(test['Embarked'].values.reshape(-1, 1))

from sklearn import preprocessing
label_encoder = preprocessing.LabelEncoder()
onehot_encoder = preprocessing.OneHotEncoder()
train['Sex'] = label_encoder.fit_transform(train['Sex'])

#train['Embarked'] = onehot_encoder.fit_transform(train['Embarked'].value.reshape(1,-1))
train.dropna()
test['Sex'] = label_encoder.fit_transform(test['Sex'])
#test['Embarked'] = onehot_encoder.fit_transform(test['Embarked'])
test.dropna()
print(test.isna().sum())
y = train.iloc[:,1].values
train = train.drop(['Survived'],axis = 1)
x = train.iloc[:,:].values

from sklearn.linear_model import LogisticRegression
regressor = LogisticRegression()
regressor.fit(x, y)
pred = regressor.predict(test)

from sklearn.metrics import accuracy_score

#print(len(test['PassengerId'])," ",len(pred))
df = pd.DataFrame({'PassengerId':test['PassengerId'],'Survived':pred})
df.to_csv("E:\ML\Kaggle Titanic//Final_Pred_new1.csv", index = False)


#AI-TECHGYM-N-C1
import numpy as np
import scipy.linalg as linalg

matrix = np.array([[1,-1,-1], [-1,1,-1], [-1,-1,1]])
display(matrix)

# 行列式
det_matrix = linalg.det(matrix)
display('行列式')
display(det_matrix)

# 逆行列
invers_matrix = linalg.inv(matrix)
display('逆行列')
display(invers_matrix)

# 固有値と固有ベクトル
eig_value, eig_vector = linalg.eig(matrix)
eig_vector = np.round(eig_vector, 2)

# 固有値と固有ベクトル
display('固有値')
display(eig_value)
display('固有ベクトル')
display(eig_vector)


import os
from celery import Celery
from celery.schedules import crontab

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'add_posts.settings')

import django

django.setup()

app = Celery('add_posts', include=['add_posts.tasks'])
app.config_from_object('django.conf:settings')
app.autodiscover_tasks()

app.conf.beat_schedule = {
    'start_update_content': {
        'task': 'add_posts.tasks.start_update_content',
        'schedule': crontab(
            hour='*/10')
    },
    'update_proxy': {
        'task': 'add_posts.tasks.update_proxy',
        'schedule': crontab(
            minute='*/10')
    },
    'delete_bad_worker_credentials': {
        'task': 'add_posts.tasks.delete_bad_worker_credentials',
        'schedule': crontab(
            minute='*/10')
    },
    'check_not_available_accounts': {
        'task': 'add_posts.tasks.check_not_available_accounts',
        'schedule': crontab(
            minute='*/50')
    },
    'start_parsing_url_new': {
        'task': 'add_posts.tasks.start_parsing_url_new',
        'schedule': crontab(
            hour='*/5')
    },
    # 'delete_old_proxy': {
    #     'task': 'add_posts.tasks.delete_old_proxy',
    #     'schedule': crontab(
    #         minute='*/10')
    # }

}


"""Linked data proof signing and verification methods."""

from typing import List

from .document_loader import DocumentLoaderMethod
from .proof_set import ProofSet
from .purposes import _ProofPurpose as ProofPurpose
from .suites import _LinkedDataProof as LinkedDataProof
from .validation_result import DocumentVerificationResult

async def sign(
    *,
    document: dict,
    suite: LinkedDataProof,
    purpose: ProofPurpose,
    document_loader: DocumentLoaderMethod,
) -> dict:
    """Cryptographically signs the provided document by adding a `proof` section.

    Proof is added based on the provided suite and proof purpose

    Args:
        document (dict): JSON-LD document to be signed.
        suite (LinkedDataProof): The linked data signature cryptographic suite
            with which to sign the document
        purpose (ProofPurpose): A proof purpose instance that will match proofs to be
            verified and ensure they were created according to the appropriate purpose.
        document_loader (DocumentLoader): The document loader to use.

    Raises:
        LinkedDataProofException: When a jsonld url cannot be resolved, OR signing fails.

    Returns:
        dict: Signed document.

    """
    return await ProofSet.add(
        document=document,
        suite=suite,
        purpose=purpose,
        document_loader=document_loader,
    )

async def verify(
    *,
    document: dict,
    suites: List[LinkedDataProof],
    purpose: ProofPurpose,
    document_loader: DocumentLoaderMethod,
) -> DocumentVerificationResult:
    """Verify the linked data signature on the provided document.

    Args:
        document (dict): The document with one or more proofs to be verified.
        suites (List[LinkedDataProof]): Acceptable signature suite instances for
            verifying the proof(s).
        purpose (ProofPurpose): A proof purpose instance that will match proofs to be
            verified and ensure they were created according to the appropriate purpose.
        document_loader (DocumentLoader): The document loader to use.

    Returns:
        DocumentVerificationResult: Object with a `verified` boolean property that is
            `True` if at least one proof matching the given purpose and suite verifies
            and `False` otherwise. a `results` property with an array of detailed
            results. if `False` an `errors` property will be present, with a list
            containing all of the errors that occurred during the verification process.

    """

    result = await ProofSet.verify(
        document=document,
        suites=suites,
        purpose=purpose,
        document_loader=document_loader,
    )

    return result

async def derive(
    *,
    document: dict,
    reveal_document: dict,
    suite: LinkedDataProof,
    document_loader: DocumentLoaderMethod,
    nonce: bytes = None,
) -> dict:
    """Derive proof(s) for document with reveal document.

    All proofs matching the signature suite type will be replaced with a derived
    proof. Other proofs will be discarded.

    Args:
        document (dict): The document with one or more proofs to be derived
        reveal_document (dict): The JSON-LD frame specifying the revealed attributes
        suite (LinkedDataProof): The linked data signature cryptographic suite
            with which to derive the proof
        document_loader (DocumentLoader): The document loader to use.
        nonce (bytes, optional): Nonce to use for the proof. Defaults to None.

    Returns:
        dict: The document with derived proof(s).

    """

    result = await ProofSet.derive(
        document=document,
        reveal_document=reveal_document,
        suite=suite,
        document_loader=document_loader,
        nonce=nonce,
    )

    return result


# ----------------------------------------------------------------------------------------------------------------------#
# Project: NURSING HOME PRESSURE ULCER
# Author: Zoey Chen
# Description: This script will preprocess MEDPAR SNF claims to merge with MDS.
# ----------------------------------------------------------------------------------------------------------------------#

import pandas as pd
import dask.dataframe as dd
import os
import numpy as np
import sys

def identify_pu(row):
    ## this function map claims to main, secondary and not_pu categories based on diagnosis codes
    ## NOTE: only main pressure ulcer claims is used in this study
    dcode = ['DGNS_{}_CD'.format(i) for i in list(range(1, 26))]

    pu1 = row[dcode[1:]].str.startswith('7070', na=False)
    pu2 = row[dcode[1:]].str.startswith('7072', na=False)
    pu3 = row[dcode[1:]].str.startswith('L89', na=False)

    ## identify main pu claims: if the admitting, the primary or the second diagnosis code is pu
    if ((row.DGNS_1_CD.startswith('7070') |  ## determine if the primary diagnosis code is pu
         row.DGNS_1_CD.startswith('7072') |
         row.DGNS_1_CD.startswith('L89')) |
            (row.DGNS_2_CD.startswith('7070') |
             row.DGNS_2_CD.startswith('7072') |  ## determine if the second diagnosis code is pu
             row.DGNS_2_CD.startswith('L89')) |
            (row.ADMTG_DGNS_CD.startswith('7070') |
             row.ADMTG_DGNS_CD.startswith('7072') |  ## determine if the admission diagnosis code is pu
             row.ADMTG_DGNS_CD.startswith('L89'))):
        row['pu_ind'] = 'main'
    elif (any(pu1) | any(pu2) | any(pu3)):
        ## if any of the secondary diagnosis codes is PU related (except those categorized as "main"),
        ## the claim is a 'spu' claims - secondary only pressure ulcer claims
        row['pu_ind'] = 'spu'
    else:
        row['pu_ind'] = 'not_pu'
    return row

def processsnf(medparpath, year, outpath):
    ## this function select PU related SNF claims and write to parquet
    ## define diagnosis columns
    dcode = ['DGNS_{}_CD'.format(i) for i in list(range(1, 26))]
    ## define diagnosis e code columns
    decode = ['DGNS_E_{}_CD'.format(i) for i in list(range(1, 13))]
    ## define poa indicator columns (not used)
    poacode = ['POA_DGNS_{}_IND_CD'.format(i) for i in list(range(1, 26))]
    ## define diagnosis version columns
    dvcode = ['DGNS_VRSN_CD_{}'.format(i) for i in list(range(1, 26))]
    dvecode = ['DGNS_E_VRSN_CD_{}'.format(i) for i in list(range(1, 13))]
    ## define columns selected for the analysis
    col_use = ['BENE_ID', 'MEDPAR_ID', 'MEDPAR_YR_NUM', 'PRVDR_NUM', 'ADMSN_DT', 'DSCHRG_DT',
               'DSCHRG_DSTNTN_CD', 'SS_LS_SNF_IND_CD', 'BENE_DSCHRG_STUS_CD', 'DRG_CD',
               'ADMTG_DGNS_CD', 'CVRD_LVL_CARE_THRU_DT'] + dcode + decode + poacode + dvcode + dvecode

    ## read in raw MedPAR data
    medpar = dd.read_parquet(medparpath, engine='fastparquet')

    medpar = medpar.reset_index()

    ## select columns and rows
    ## 1) select inpatient data
    ## 2) select columns
    ## 3) select pu related claims

    ## 1) select SNF claims
    snf = medpar[medpar.SS_LS_SNF_IND_CD == 'N']
    ## 2) select columns
    snf = snf[col_use]

    ## create an indicator denotes the claim is a medpar claim
    snf['r'] = 'h'
    ## identify main and secondary pressure ulcer claims
    snf = snf.apply(identify_pu, axis=1)
    snf_main = snf[snf['pu_ind'] == 'main']
    snf_spu = snf[snf['pu_ind'] == 'spu']

    snf_main = snf_main.drop(columns='pu_ind')
    snf_spu = snf_spu.drop(columns='pu_ind')
    ## write pu SNF claims to parquet
    snf_main.to_parquet(outpath + 'main_pu_claims_snf/{}/'.format(year))
    snf_spu.to_parquet(outpath + 'secondary_only_pu_claims_snf/{}/'.format(year))

    return snf_main

def merge_mbsf(input_snf, input_mbsf, output_path):
    ## read in SNF pu claims
    snf = dd.read_parquet(input_snf)
    ## read in MBSF base data
    mbsf = dd.read_csv(input_mbsf,
                       dtype={'ESRD_IND': 'object'},
                       low_memory=False,
                       assume_missing=True
                       )
    ## define dual and hmo columns
    dual = ['DUAL_STUS_CD_{:02d}'.format(i) for i in range(1, 13)]
    hmo  = ['HMO_IND_{:02d}'.format(i) for i in range(1, 13)]
    col_use = ['BENE_ID', 'BENE_DEATH_DT', 'ESRD_IND', 'AGE_AT_END_REF_YR',
               'RTI_RACE_CD', 'BENE_ENROLLMT_REF_YR', 'SEX_IDENT_CD',
               'ENTLMT_RSN_ORIG', 'ENTLMT_RSN_CURR'] + dual + hmo
    ## select columns
    mbsf = mbsf[col_use]
    mbsf = mbsf.rename(columns={'BENE_DEATH_DT': 'death_dt',
                              'AGE_AT_END_REF_YR': 'age',
                              'RTI_RACE_CD': 'race_RTI',
                              'SEX_IDENT_CD': 'sex'})

    ## include only FFS for the full year
    mbsf = mbsf[mbsf[hmo].isin(['0', '4']).all(axis=1)]
    ## merge mbsf with snf pu claims
    merge = snf.merge(mbsf, on='BENE_ID')
    merge.to_parquet(output_path)
    return merge

if __name__ == '__main__':
    import yaml
    from dask.distributed import Client
    client = Client("10.50.86.250:55317")

    # input and output paths
    years = list(range(2011, 2018))
    yaml_path = '/gpfs/data/sanghavi-lab/Zoey/gardner/nhc_pressure_ulcer/final_code/'
    path = yaml.safe_load(open(yaml_path + 'data_path.yaml'))

    ## RUN THE FUNCTIONs TO PROCESS MEDPAR SNF Claims
    for y in range(len(years)):
        snf_main = processsnf(path['processSNF']['input'][y], years[y], outpath=path['processSNF']['output'])
    ## RUN THE FUNCTION TO MERGE MBSF WITH SNF PU CLAIMS
    for y in range(len(years)):
        snf_main_merge = merge_mbsf(input_snf=path['processSNF']['output'] + 'main_pu_claims_snf/{}/'.format(years[y]),
                                    input_mbsf=path['processSNF']['input_mbsf'][y],
                                    output_path=path['processSNF']['output'] + 'main_pu_claims_snf_ffs/{}/'.format(years[y]))

    # ## flowchart calculation
    # ## total SNF claims
    # n_snf = []
    # for p in path['processSNF']['input']:
    #     df = dd.read_parquet(p)
    #     df = df[df.SS_LS_SNF_IND_CD=="N"]
    #     n_snf.append(df.shape[0].compute())
    # print(sum(n_snf))  # 125,019,398
    #
    ## total SNF FFS pressure ulcer claims
    ## main: 138780
    n_pu_snf = []
    for y in years:
        df = dd.read_parquet(path['processSNF']['output'] + 'main_pu_claims_snf_ffs/' + str(y))
        n_pu_snf.append(df.shape[0].compute())
    print(sum(n_pu_snf))


import json
import os
from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch

from flask_testing import TestCase  # type: ignore

class TestApi(TestCase):
    def create_app(self):
        from registration_ref.app import app

        return app

    def _sign(self, data: dict):
        headers = {"Content-type": "application/json"}
        return self.client.post("/sign", headers=headers, data=json.dumps(data))

    def test_no_data(self):
        """Fail properly when no data is sent"""
        r = self.client.post("/sign")
        self.assertEqual(400, r.status_code, r.data)
        self.assertIn(b"Missing request body", r.data)

    def test_no_csr(self):
        """Fail properly if no csr is provided"""
        r = self._sign({"data": 12})
        self.assertEqual(400, r.status_code, r.data)
        self.assertIn(b"Missing required field 'csr'", r.data)

    def test_invalid_csr(self):
        r = self._sign({"csr": "not a valid csr"})
        self.assertEqual(400, r.status_code, r.data)
        self.assertIn(b"Unable to load request", r.data)

    @patch("registration_ref.app.sign_device_csr")
    @patch("registration_ref.settings._env")
    def test_overrides(self, env, sign_device_csr):
        """Ensure we return proper sota toml overrides"""
        env.return_value = "https://example.com:8443"
        fields = Mock()
        fields.root_crt = "root_crt"
        fields.client_crt = "client_crt"
        fields.uuid = "uuid"
        fields.pubkey = "pub"
        sign_device_csr.return_value = fields
        overrides = {
            "pacman": {
                "type": "ostree_foo",
            }
        }
        with TemporaryDirectory() as d:
            with patch("registration_ref.app.Settings") as settings:
                settings.DEVICES_DIR = d
                settings.API_TOKEN_PATH = None
                r = self._sign({"csr": "n/a", "overrides": overrides})
                with open(os.path.join(d, "uuid")) as f:
                    self.assertEqual("pub", f.read())
        self.assertEqual(201, r.status_code, r.data)
        toml = r.json["sota.toml"]
        self.assertIn("[pacman]\ntype = ostree_foo", toml)
        self.assertIn('[provision]\nserver = "https://example.com:8443"', toml)


import logging

from blinker import signal

from jarr.metrics import EVENT

event = signal('jarr-event')

@event.connect
def bump_metric(sender, **kwargs):
    EVENT.labels(**{key: kwargs[key] for key in ('module', 'context',
                                                 'result')}).inc()

@event.connect
def log(sender, level=logging.DEBUG, **kwargs):
    logger = logging.getLogger(kwargs['module'])
    logger.info("%s > %r: %s",
                kwargs['module'], kwargs["context"], kwargs["result"])


import glob
import os
import signal
import subprocess
from typing import Tuple, Optional

from .util import LOG
from .parsing_bjobs import parse_run_time

# TODO: load and display job output while running
#   open the files and keep loading the data while available
#   or ideally run `less` in a subwindow?

N_PREVIEW_LINES = 15

class OutputViewer:
    LSBATCH_DIR = "/cluster/shadow/.lsbatch"

    def __init__(self):
        self.id_to_file = {}

    @staticmethod
    def _find_running_output_file(job_id):
        """Does not try to use cached results."""
        # LOG.append(f"Globbing for {job_id}")
        candidates = glob.glob(f"/cluster/shadow/.lsbatch/*{job_id}.out")
        if len(candidates) > 1:
            LOG.append(f"Warning: more than one output file found? {candidates}")

        if candidates:
            return candidates[0]
        else:
            return None

    def get_running_output_file(self, job_id):
        if job_id in self.id_to_file:
            # This can also be None, if we've tried to find this before and didn't succeed
            return self.id_to_file[job_id]
        else:
            res = self._find_running_output_file(job_id)
            # again - possibly None!
            self.id_to_file[job_id] = res

            return res

    def get_finished_output_file(self, job):
        if job.get("exec_cwd") and job.get("output_file"):
            return os.path.join(job["exec_cwd"], job["output_file"])
        else:
            return None

    def get_output_file(self, job):
        # See https://www.ibm.com/docs/en/spectrum-lsf/10.1.0?topic=execution-about-job-states
        # for info about job states

        if job["stat"] in ["RUN", "USUSP", "SSUSP"]:
            run_time = parse_run_time(job["run_time"])

            if run_time and run_time >= 10:
                return self.get_running_output_file(job["jobid"])
            else:
                # For the first few seconds of a job's runtime,
                # the file might not exist yet.
                return None
        elif job["stat"] in ["PEND", "PSUSP"]:
            return None
        elif job["stat"] in ["DONE", "EXIT"]:
            return self.get_finished_output_file(job)
        else:
            LOG.append(f'Unknown job status {job["stat"]}')
            return None

    def get_output_preview(
        self, job, n_preview_lines=N_PREVIEW_LINES
    ) -> Tuple[Optional[str], str]:
        if job is None:
            # Happens when there are no jobs at all.
            return None, "(no jobs)"

        output_file = self.get_output_file(job)
        if output_file is None:
            return None, f"(no output file available for job {job['jobid']})"

        try:
            with open(output_file, "rb") as f:
                output_preview = tail(f, lines=n_preview_lines)

            return output_file, output_preview
        except FileNotFoundError:
            return output_file, f"Couldn't find {output_file}"

    def open_output_fullscreen(self, job):
        output_file = self.get_output_file(job)
        if output_file:
            old_action = signal.signal(signal.SIGINT, signal.SIG_IGN)

            command = ["less", "-r"]

            if job["stat"] == "RUN":
                # Tail the file (wait for incoming data) if the job is still running
                command.append("+F")
            else:
                # Jump to the end.
                command.append("+G")

            subprocess.run(command + [output_file])
            signal.signal(signal.SIGINT, old_action)

def tail(f, lines=20):
    # https://stackoverflow.com/questions/136168/get-last-n-lines-of-a-file-similar-to-tail
    total_lines_wanted = lines

    BLOCK_SIZE = 1024
    f.seek(0, 2)  # seek to the end
    block_end_byte = f.tell()
    lines_to_go = total_lines_wanted
    block_number = -1
    blocks = []
    while lines_to_go > 0 and block_end_byte > 0:
        if block_end_byte - BLOCK_SIZE > 0:
            f.seek(block_number * BLOCK_SIZE, 2)
            blocks.append(f.read(BLOCK_SIZE))
        else:
            # Can read the entirety of the file
            f.seek(0, 0)
            blocks.append(f.read(block_end_byte))
        lines_found = blocks[-1].count(b"\n")
        lines_to_go -= lines_found
        block_end_byte -= BLOCK_SIZE
        block_number -= 1
    all_read_text = b"".join(reversed(blocks))

    res_bytes = b"\n".join(all_read_text.splitlines()[-total_lines_wanted:])
    return res_bytes.decode("utf-8", errors="replace")


# GENERAL IMPORTS
from string import digits
from random import choice
from time import sleep
from selenium.webdriver.chrome.webdriver import WebDriver

# CUSTOM IMPORTS
from config import (
    bot_quick_name_url,
    bot_quick_name_classname,
    bot_username_url,
    bot_username_search_button_xPath,
    bot_username_search_field_xPath,
)

# FUNCTION TO FETCH RANDOM FULL NAME FROM ONLINE NAME GENERATOR,
# AND RETURN FIRST AND LAST NAME
def fetch_name(browser: WebDriver) -> tuple[str, str]:
    browser.get(bot_quick_name_url)
    name_element = browser.find_element_by_class_name(bot_quick_name_classname)
    full_name: str = name_element.text
    full_name_split: list = full_name.split()
    first_name: str = full_name_split[0]
    last_name: str = full_name_split[1]
    return (first_name, last_name)

# FUNCTION TO FETCH RANDOM USERNAME FROM ONLINE USERNAME GENERATOR,
# AND RETURN USERNAME
def fetch_username(browser: WebDriver, first_name: str) -> str:
    browser.get(bot_username_url)
    search_box = browser.find_element_by_xpath(bot_username_search_field_xPath)
    search_box.click()
    search_box.send_keys(f"{first_name}")
    sleep(0.5)
    browser.find_element_by_xpath(bot_username_search_button_xPath).click()
    user_name: str = browser.find_element_by_xpath(
        f'//*[contains(@id, "{first_name}")]'
    ).text
    return user_name

# FUNCTION TO FETCH MULTIPLE USERNAMES FROM ONLINE USERNAME GENERATOR,
# JOINING RANDOM USERNAMES IN RANDOM ORDER,
# AND RETURN FINAL USERNAME
def create_user_name(browser: WebDriver, first_name: str) -> str:
    nDigits: int = 4
    user_name_raw: str = fetch_username(browser=browser, first_name=first_name)
    user_name_extras: str = digits
    user_name_extra_list: list[str] = []
    for _ in range(nDigits):
        user_name_extra_list.append(choice(user_name_extras))
    user_name_extra_string: str = "".join(user_name_extra_list)
    user_name_final: str = user_name_raw + user_name_extra_string
    return user_name_final


# coding: utf-8

# In[1]:

import sys
import sklearn
import matplotlib
import numpy as np

print('Python: {}'.format(sys.version))
print('Sklearn: {}'.format(sklearn.__version__))
print('Matplotlib: {}'.format(matplotlib.__version__))
print('NumPy: {}'.format(np.__version__))

# In[4]:

from keras.datasets import mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()

# In[5]:

print('Training Data: {}'.format(x_train.shape))
print('Training Labels: {}'.format(y_train.shape))
print('Testing Data: {}'.format(x_test.shape))
print('Testing Labels: {}'.format(y_test.shape))

# In[6]:

import matplotlib.pyplot as plt

# python magic function
get_ipython().magic('matplotlib inline')

# In[7]:

# create figure with 3x3 subplots using matplotlib.pyplot
fig, axs = plt.subplots(3, 3, figsize = (12, 12))
plt.gray()

# loop through subplots and add mnist images
for i, ax in enumerate(axs.flat):
    ax.matshow(x_train[i])
    ax.axis('off')
    ax.set_title('Number {}'.format(y_train[i]))

# display the figure
fig.show()

# In[8]:

# convert each image to 1 dimensional array
X = x_train.reshape(len(x_train),-1)
Y = y_train

# normalize the data to 0 - 1
X = X.astype(float) / 255.

print(X.shape)
print(X[0].shape)

# In[10]:

from sklearn.cluster import MiniBatchKMeans

n_digits = len(np.unique(y_test))
print(n_digits)

# Initialize KMeans model
kmeans = MiniBatchKMeans(n_clusters = n_digits)

# Fit the model to the training data
kmeans.fit(X)

# In[11]:

kmeans.labels_

# In[27]:

def infer_cluster_labels(kmeans, actual_labels):
    """
    Associates most probable label with each cluster in KMeans model
    returns: dictionary of clusters assigned to each label
    """

    inferred_labels = {}

    for i in range(kmeans.n_clusters):

        # find index of points in cluster
        labels = []
        index = np.where(kmeans.labels_ == i)

        # append actual labels for each point in cluster
        labels.append(actual_labels[index])

        # determine most common label
        if len(labels[0]) == 1:
            counts = np.bincount(labels[0])
        else:
            counts = np.bincount(np.squeeze(labels))

        # assign the cluster to a value in the inferred_labels dictionary

        if np.argmax(counts) in inferred_labels:
            # append the new number to the existing array at this slot
            inferred_labels[np.argmax(counts)].append(i)
        else:
            # create a new array in this slot
            inferred_labels[np.argmax(counts)] = [i]
#         print('Cpunt : {} ' .format(np.argmax(counts)))
#         print('Cpunt : {} ' .format(counts))
#         print(labels)
#         print('Cluster: {}, label: {}'.format(i, np.argmax(counts)))

    return inferred_labels

def infer_data_labels(X_labels, cluster_labels):
    """
    Determines label for each array, depending on the cluster it has been assigned to.
    returns: predicted labels for each array
    """

    # empty array of len(X)
    predicted_labels = np.zeros(len(X_labels)).astype(np.uint8)

    for i, cluster in enumerate(X_labels):
        for key, value in cluster_labels.items():
            if cluster in value:
                predicted_labels[i] = key

    return predicted_labels

# In[30]:

# test the infer_cluster_labels() and infer_data_labels() functions
cluster_labels = infer_cluster_labels(kmeans, Y)
print(cluster_labels)
X_clusters = kmeans.predict(X)
predicted_labels = infer_data_labels(X_clusters, cluster_labels)
print (predicted_labels[:20])
print (Y[:20])

# In[33]:

from sklearn import metrics

def calculate_metrics(estimator, data, labels):

    # Calculate and print metrics
    print('Number of Clusters: {}'.format(estimator.n_clusters))
    print('Inertia: {}'.format(estimator.inertia_))
    print('Homogeneity: {}'.format(metrics.homogeneity_score(labels, estimator.labels_)))

# In[34]:

clusters = [10, 16, 36, 64, 144, 256]

# test different numbers of clusters
for n_clusters in clusters:
    estimator = MiniBatchKMeans(n_clusters = n_clusters)
    estimator.fit(X)

    # print cluster metrics
    calculate_metrics(estimator, X, Y)

    # determine predicted labels
    cluster_labels = infer_cluster_labels(estimator, Y)
    predicted_Y = infer_data_labels(estimator.labels_, cluster_labels)

    # calculate and print accuracy
    print('Accuracy: {}\n'.format(metrics.accuracy_score(Y, predicted_Y)))

# In[35]:

# test kmeans algorithm on testing dataset
# convert each image to 1 dimensional array
X_test = x_test.reshape(len(x_test),-1)

# normalize the data to 0 - 1
X_test = X_test.astype(float) / 255.

# initialize and fit KMeans algorithm on training data
kmeans = MiniBatchKMeans(n_clusters = 256)
kmeans.fit(X)
cluster_labels = infer_cluster_labels(kmeans, Y)

# predict labels for testing data
test_clusters = kmeans.predict(X_test)
predicted_labels = infer_data_labels(kmeans.predict(X_test), cluster_labels)

# calculate and print accuracy
print('Accuracy: {}\n'.format(metrics.accuracy_score(y_test, predicted_labels)))

# In[36]:

# Initialize and fit KMeans algorithm
kmeans = MiniBatchKMeans(n_clusters = 36)
kmeans.fit(X)

# record centroid values
centroids = kmeans.cluster_centers_

# reshape centroids into images
images = centroids.reshape(36, 28, 28)
images *= 255
images = images.astype(np.uint8)

# determine cluster labels
cluster_labels = infer_cluster_labels(kmeans, Y)

# create figure with subplots using matplotlib.pyplot
fig, axs = plt.subplots(6, 6, figsize = (20, 20))
plt.gray()

# loop through subplots and add centroid images
for i, ax in enumerate(axs.flat):

    # determine inferred label using cluster_labels dictionary
    for key, value in cluster_labels.items():
        if i in value:
            ax.set_title('Inferred Label: {}'.format(key))

    # add image to subplot
    ax.matshow(images[i])
    ax.axis('off')

# display the figure
fig.show()


import cbsodata
import ssl
import streamlit as st
import geopandas as gpd
import pyproj
import pandas as pd
import json
import numpy as np

mapboxt = 'pk.eyJ1IjoiZG9taW5pcXVlLXZkdiIsImEiOiJjbDNrYm4wbHEwNWNwM2RyczR0a2UxeXRoIn0.e-S9dzdYyw7IW6_B-WMHVA'
ssl._create_default_https_context = ssl._create_unverified_context

def get_cbs_dummydata():
    # Zoek op welke data beschikbaar is
    metadata = pd.DataFrame(cbsodata.get_meta('83765NED', 'DataProperties'))

    # Download data en verwijder spaties uit regiocodes
    data = pd.DataFrame(cbsodata.get_data('83765NED', select = ['WijkenEnBuurten','Gemeentenaam_1',"Omgevingsadressendichtheid_105", 'Codering_3', 'AantalInwoners_5']))
    data['Codering_3'] = data['Codering_3'].str.strip()
    data = data.rename(columns={'Codering_3': 'REGIO_CODE'})

    return data

def get_data():
    aantal_inwoners_per_provincie = pd.read_csv("data/indicatoren/aantalinw_totaal_65+_2020_provincie.csv", sep=';').rename(
        columns={'Mannen en vrouwen (aantal)': 'aantal_inwoners', "Regio's": "REGIO_NAAM"}).assign(
        REGIO_NAAM=lambda d: [el.replace(" (PV)", "") for el in d.REGIO_NAAM]).drop(
        columns=['Perioden', 'Geboorteland'])
    totaal = aantal_inwoners_per_provincie.loc[lambda d: d.Leeftijd == 'Totaal'].rename(columns={'aantal_inwoners': 'aantal_inwoners_alle_leeftijd'}).drop(columns=["Leeftijd"])
    ouder = aantal_inwoners_per_provincie.loc[lambda d: d.Leeftijd == '65 jaar of ouder'].rename(columns={'aantal_inwoners': 'aantal_inwoners_65_plus'}).drop(columns=["Leeftijd"])
    aantal_inwoners_per_provincie = totaal.merge(ouder, on=['REGIO_NAAM'])

    arbeidsvolume_huisartsen_per_provincie = pd.read_csv("data/indicatoren/arbeidsvolume_huisartsen_gzc_provincie_2020.csv", sep=';').drop(
        columns=['Perioden', 'AZW branches']).rename(
        columns={'Arbeidsvolume  (x 1 000)': 'aantal_fte', "Regio's": "REGIO_NAAM"}).assign(REGIO_NAAM = lambda d: [el.replace(" (PV)", "") for el in d.REGIO_NAAM], aantal_fte = lambda d: [float(el.replace(',','.')) *1000 for el in d.aantal_fte])

    df_provincie = arbeidsvolume_huisartsen_per_provincie.merge(aantal_inwoners_per_provincie, on=['REGIO_NAAM'])
    df_provincie = df_provincie.assign(aantal_fte_per_1000 = lambda d: np.round(d.aantal_fte / d.aantal_inwoners_alle_leeftijd * 1000, 3))

    ervarengezondheid = (pd.read_csv("data/indicatoren/ervarengezondheid_65plus_2020_gemeente.csv", sep=';')
                         .drop(columns=['ID', 'Persoonskenmerken', 'Marges']).rename(columns={'RegioS': 'REGIO_CODE'})
                         .assign(ErvarenGezondheidGoedZeerGoed_6=lambda d: [el.strip() for el in d.ErvarenGezondheidGoedZeerGoed_6])
                         .assign(ErvarenGezondheidGoedZeerGoed_6 = lambda d: np.where(d.ErvarenGezondheidGoedZeerGoed_6 == '.', np.nan, d.ErvarenGezondheidGoedZeerGoed_6))
                         .assign(ErvarenGezondheidGoedZeerGoed_6 = lambda d: [float(el) for el in d.ErvarenGezondheidGoedZeerGoed_6])
                         )
    instroomleeftijd_wlz = pd.read_csv("data/indicatoren/instroomleeftijdwlz_gemeente_2020.csv", sep=';').drop(columns=['Geslacht']).rename(columns={"Regio's":"REGIO_NAAM","Wmo-cliënten; gebruik Wlz aantal (aantal)":"aantal_wlz_gebruikers" })

    totaal_aantal_wlz_gebruikers = instroomleeftijd_wlz.groupby('REGIO_NAAM').sum().assign(mediaan=lambda d: d.aantal_wlz_gebruikers/2)
    cumulative = instroomleeftijd_wlz.sort_values(["REGIO_NAAM", "Persoonskenmerken"]).set_index(
        ['REGIO_NAAM', 'Persoonskenmerken']).groupby('REGIO_NAAM').cumsum().reset_index()

    df1 = cumulative.loc[lambda d: d['Persoonskenmerken'] == 'Leeftijd: 45 tot 60 jaar'].drop(
        columns=['Persoonskenmerken']).rename(columns={'aantal_wlz_gebruikers': 'aantal_wlz_gebruikers_45_60'})
    df2 = cumulative.loc[lambda d: d['Persoonskenmerken'] == 'Leeftijd: 60 tot 75 jaar'].drop(
        columns=['Persoonskenmerken']).rename(columns={'aantal_wlz_gebruikers': 'aantal_wlz_gebruikers_60_75'})
    df3 = cumulative.loc[lambda d: d['Persoonskenmerken'] == 'Leeftijd: 75 tot 85 jaar'].drop(
        columns=['Persoonskenmerken']).rename(columns={'aantal_wlz_gebruikers': 'aantal_wlz_gebruikers_75_85'})
    df4 = cumulative.loc[lambda d: d['Persoonskenmerken'] == 'Leeftijd: 85 jaar of ouder'].drop(
        columns=['Persoonskenmerken']).rename(columns={'aantal_wlz_gebruikers': 'aantal_wlz_gebruikers_85_plus'})
    wlz_gebruikers_alles = df1.merge(df2, on='REGIO_NAAM').merge(df3, on="REGIO_NAAM").merge(df4, on="REGIO_NAAM").merge(totaal_aantal_wlz_gebruikers.reset_index().drop(columns=['aantal_wlz_gebruikers']), on='REGIO_NAAM')
    instroomleeftijd_wlz = wlz_gebruikers_alles.assign(mediaan_instroomleeftijd_wlz = lambda d: np.where(
        d.mediaan < d.aantal_wlz_gebruikers_45_60, 45 + np.round(d.mediaan / (d.aantal_wlz_gebruikers_45_60/15)),
        np.where(
            d.mediaan < d.aantal_wlz_gebruikers_60_75, 60 + np.round((d.mediaan-d.aantal_wlz_gebruikers_45_60) / (d.aantal_wlz_gebruikers_60_75/15)),
            np.where(
                d.mediaan < d.aantal_wlz_gebruikers_75_85, 75+np.round((d.mediaan - d.aantal_wlz_gebruikers_60_75) / (d.aantal_wlz_gebruikers_75_85 / 10)),
                85+np.round((d.mediaan - d.aantal_wlz_gebruikers_75_85) / (d.aantal_wlz_gebruikers_85_plus/10))
            )
        )
    ))[['REGIO_NAAM', 'mediaan_instroomleeftijd_wlz']]

    crosswalk = pd.read_csv("data/GM_naar_GGD_en_PV.csv", sep=';')
    crosswalk_gemeente = crosswalk[["Codes en namen van gemeenten/Code (code)", "Codes en namen van gemeenten/Naam (naam)"]]
    crosswalk_gemeente.columns = ['REGIO_CODE','REGIO_NAAM']
    crosswalk_gemeente = crosswalk_gemeente.assign(REGIO_CODE= lambda d: [el.strip() for el in d.REGIO_CODE],
                              REGIO_NAAM= lambda d: [el.strip() for el in d.REGIO_NAAM])
    crosswalk_provincie = crosswalk[['Lokaliseringen van gemeenten/Provincies/Code (code)',
                                     'Lokaliseringen van gemeenten/Provincies/Naam (naam)']]
    crosswalk_provincie=crosswalk_provincie.loc[~crosswalk_provincie.duplicated()]
    crosswalk_provincie.columns = ['REGIO_CODE', 'REGIO_NAAM']
    crosswalk_provincie = crosswalk_provincie.assign(REGIO_CODE=lambda d: [el.strip() for el in d.REGIO_CODE],
                                                   REGIO_NAAM=lambda d: [el.strip() for el in d.REGIO_NAAM])

    instroomleeftijd_wlz = instroomleeftijd_wlz.merge(crosswalk_gemeente, on='REGIO_NAAM') # hier vallen er een paar weg
    ervarengezondheid = ervarengezondheid.merge(crosswalk_gemeente, on='REGIO_CODE')
    df_gemeente = ervarengezondheid.merge(instroomleeftijd_wlz, on=["REGIO_CODE", "REGIO_NAAM"])
    df_gemeente = df_gemeente.rename(columns={'ErvarenGezondheidGoedZeerGoed_6': 'Ervaren gezondheid (Goed / zeer goed, %)',
                                                "mediaan_instroomleeftijd_wlz": "Instroomleeftijd wlz (mediaan)"})

    df_provincie = df_provincie.merge(crosswalk_provincie, on="REGIO_NAAM").assign(REGIO_NAAM=lambda d: np.where(d.REGIO_NAAM == "Fryslân", "Friesland", d.REGIO_NAAM))
    df_provincie = df_provincie.rename(columns={'aantal_fte_per_1000': 'Aantal huisartsen FTEs per 1000 inwoners', "aantal_inwoners_65_plus": "Aantal 65+'ers"})

    return df_gemeente, df_provincie

def get_cbs_polygons():
    # Haal de kaart met gemeentegrenzen op van PDOK
    geodata_url = 'https://geodata.nationaalgeoregister.nl/cbsgebiedsindelingen/wfs?request=GetFeature&service=WFS&version=1.1.0&outputFormat=application%2Fjson&typeName=cbsgebiedsindelingen:cbs_gemeente_2020_gegeneraliseerd'
    gemeentegrenzen = gpd.read_file(geodata_url)
    gemeentegrenzen = gemeentegrenzen.rename(columns={'statcode': 'REGIO_CODE', 'statnaam': 'REGIO_NAAM'})

    geodata_url_ggd ="https://geodata.nationaalgeoregister.nl/cbsgebiedsindelingen/wfs?request=GetFeature&service=WFS&version=1.1.0&outputFormat=application%2Fjson&typeName=cbsgebiedsindelingen:cbs_provincie_2020_gegeneraliseerd"
    provinciegrenzen = gpd.read_file(geodata_url_ggd)
    provinciegrenzen = provinciegrenzen.rename(columns={'statcode': 'REGIO_CODE', 'statnaam': 'REGIO_NAAM'})
    return gemeentegrenzen, provinciegrenzen

def merge_data_with_polygons(geometries, dataframe: pd.DataFrame, merge_col = "REGIO_NAAM"):
    # Koppel CBS-data aan geodata met regiocodes
    map_df = pd.merge(geometries,dataframe, on = merge_col)

    # maak kaart
    map_df.to_crs(pyproj.CRS.from_epsg(4326), inplace=True)

    return map_df

def create_geojson(data_and_geometries: pd.DataFrame, filepath=r'geojson.json', id="REGIO_NAAM"):

    # # write GeoJSON to filea
    data_and_geometries.to_file(filepath, driver="GeoJSON")

    with open(filepath) as geofile:
        j_file = json.load(geofile)

    # index geojson (st id value is equal to area code
    i = 1
    for feature in j_file["features"]:
        feature['id'] = j_file['features'][i - 1]['properties'][id]
        i += 1

    return j_file


from mrjob.job import MRJob
from mrjob.step import MRStep

class TempDataSet(MRJob):
    def steps(self):
        return [
            MRStep(mapper = self.get_min_temps, reducer =self.reducer_minimum)
        ]

    def get_min_temps(self, _ , line):
        wheather_station, _, obs_type, temp,_,_,_,_= line.split(",")
        if wheather_station in ["ITE00100554", "EZE00100082"] and obs_type=="TMIN":
            yield  wheather_station, float(temp)

    def reducer_minimum(self, key, values):
        yield key, min(values)/10

if __name__=="__main__":
    TempDataSet.run()


import pysam
from sys import stdin, stdout, exit
from os import path
import pybedtools
import numpy as np
from athena.utils.misc import bgzip as bgz
from athena.utils.math import hwe_chisq

def filter_vcf(vcf, out, chroms, xchroms, svtypes, exclusion_list,
               minAF, maxAF, minAC, maxAC, minAN, filters,
               minQUAL, maxQUAL, HWE, af_fields, keep_infos, bgzip):

    # Open connection to input VCF
    if vcf in '- stdin'.split():
        invcf = pysam.VariantFile(stdin)
    else:
        invcf = pysam.VariantFile(vcf)
    header = invcf.header

    #Clean undesired INFO fields from header
    if keep_infos != 'ALL':
        if keep_infos is None:
            keep_infos = []
        else:
            keep_infos = keep_infos.split(',')
        for key in 'END CHR2 SVTYPE SVLEN'.split():
            keep_infos.append(key)
        for key in header.info.keys():
            if key not in keep_infos:
                header.info.remove_header(key)

    # Open connection to output VCF
    if out in '- stdout'.split():
        outvcf = pysam.VariantFile(stdout, 'w', header=header)
    else:
        if '.gz' in out:
            out = path.splitext(out)[0]
        outvcf = pysam.VariantFile(out, 'w', header=header)

    # Parse filtering options
    if chroms is not None:
        chroms = chroms.split(',')
    else:
        chroms = header.contigs.keys()
    if xchroms is not None:
        xchroms = xchroms.split(',')
        chroms = [c for c in chroms if c not in xchroms]
    if svtypes is not None:
        if 'SVTYPE' not in header.info.keys():
            sys.exit('SVTYPE filtering was specified, but input VCF ' +
                     'does not have SVTYPE entry in INFO.')
        else:
            svtypes = svtypes.split(',')
    if filters is not None:
        filters = filters.split(',')
    if exclusion_list is not None:
        bl = pybedtools.BedTool(exclusion_list)

    # Raise warning if AF or AC are missing from VCF
    for key in af_fields + ['AC']:
        if key not in header.info.keys():
            import warnings
            warning_message = '{0} not found in VCF INFO, so {0}-based filtering ' + \
                              'will be ignored'
            warning_message = warning_message.format(key)
            warnings.warn(warning_message, RuntimeWarning)

    # Raise exception if HWE enabled but any necessary fields missing
    if HWE is not None:
        for key in 'N_HOMREF N_HET N_HOMALT'.split():
            if key not in header.info.keys():
                error_message = 'Hardy-Weinberg filtering not possible due to ' + \
                                'missing {0} in VCF INFO'
                sys.exit(error_message.format(key))

    # Iterate over vcf & filter records
    for record in invcf.fetch():
        # Filter by chromosome
        if chroms is not None \
        and record.chrom not in chroms:
            continue

        # Filter by svtype
        if svtypes is not None \
        and record.info['SVTYPE'] not in svtypes:
            continue

        # Exclude records where end < start
        if record.stop < record.start:
            continue

        # Filter by AF/AC
        for af_field in af_fields:
            if af_field in record.info.keys():
                if minAF is not None:
                    if np.nansum(record.info[af_field]) < minAF:
                        continue
            if maxAF is not None:
                if np.nansum(record.info[af_field]) > maxAF:
                    continue
        if 'AC' in record.info.keys():
            if minAC is not None:
                if np.nansum(record.info['AC']) < minAC:
                    continue
            if maxAC is not None:
                if np.nansum(record.info['AC']) > maxAC:
                    continue

        # Filter by AN
        if 'AN' in record.info.keys():
            if minAN is not None:
                if record.info['AN'] < minAN:
                    continue

        # Filter by VCF FILTER
        if filters is not None:
            if len([f for f in record.filter if f not in filters]) > 0:
                continue

        # Filter by QUAL score
        if minQUAL is not None \
        and record.qual is not None:
            if record.qual < minQUAL:
                continue
        if maxQUAL is not None \
        and record.qual is not None:
            if record.qual > maxQUAL:
                continue

        # Filter by Hardy-Weinberg equilibrium
        if HWE is not None and len(record.alts) < 3:
            if np.nansum(record.info[af_field]) < 1:
                if hwe_chisq(record) < HWE:
                    continue

        # Clean record
        if keep_infos != 'ALL':
            for key in record.info.keys():
                if key not in keep_infos:
                    record.info.pop(key)

        # Write filter-passing records to output VCF
        outvcf.write(record)

    outvcf.close()

    # Filter remaining records against exclusion_list
    if exclusion_list is not None:
        prebl_vcf = pybedtools.BedTool(out)
        prebl_vcf.intersect(bl, header=True, v=True).saveas(out)

    # Bgzip output VCF, if optioned
    if bgzip:
        bgz(out)


import os, time, re, pickle, signal
import pytesseract
from PIL import Image
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import TimeoutException
from selenium import webdriver
import scrapy
import json
from postscrape.items import PostscrapeItem
import openpyxl
import random

class DBDSpider(scrapy.Spider):
    name = "dbd_thai"
    # start_urls = ['https://datawarehouse.dbd.go.th/company/profile/5/0105554123553']
    headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
    }
    allowed_domains = ["datawarehouse.dbd.go.th"]

    def start_requests(self):
        companies_id = self.random_company()
        for i in companies_id:
            # company_id = i[0]
            url = 'https://datawarehouse.dbd.go.th/company/profile/%s/%s' %(i[3],i)
            yield scrapy.Request(url, self.parse)
            # yield scrapy.Request(url=url, cookies={"JSESSIONID":sef.getCookie}, callback=self.parse)

    def getCookie(self):
        cookie_path = '/Users/mya/Desktop/Development/scrapyTest/postscrape/postscrape/spiders/temp/cookie.json'
        if os.path.isfile(cookie_path):
            try:
                with open(cookie_path, 'rb') as f:
                    cookies = pickle.load(f)
            except EOFError:
                cookies = None

        for i in cookies:
            if i['name']=='JSESSIONID':
                cookies= i['value']
                break
        print(cookies)
        return cookies

    # def writeJsonFile(self, data):
    #     filePath = '/Users/mya/Desktop/Development/scrapyTest/postscrape/postscrape/spiders/temp/thaiVersion.json'
    #     a_file = open(filePath, "w", encoding='utf-8')
    #     line = json.dumps(data, ensure_ascii=False) + "\n"
    #     a_file.write(line)

    # def readLoadsFile(self):
    #     loadsfilePath = '/Users/mya/Desktop/Development/scrapyTest/postscrape/postscrape/spiders/temp/thaiVersion.json'
    #     print('------------Target Company Information------------')
    #     loadsdata = json.load(open(loadsfilePath))
    #     print(loadsdata)
    #     return loadsdata

    def random_company(self):
        companies_path = '/Users/mya/Desktop/Development/scrapyTest/postscrape/postscrape/spiders/db/dbd_oct2020.xlsx'
        companies_id = []
        wb_obj = openpyxl.load_workbook(companies_path)
        sheet_obj = wb_obj.active
        max_row = sheet_obj.max_row
        for i in range(5):
            rnum = random.randint(4, max_row + 1)
            cell_obj = sheet_obj.cell(row = rnum, column = 2)
            companies_id.append(cell_obj.value)
        print(companies_id)
        return companies_id

    def parse(self, response):
        print('------------START SCRAPING------------')
        time.sleep(5)
        objective = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[3]/div[5]/div/p/text()').get()
        if objective == '-' or objective == None:
            objective = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[3]/div[3]/div/p/text()').get()

        director_list = []
        directors = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[2]/div/div/ol/li/text()').getall()
        for i in directors:
            director_list.append(i.strip())

        raw_bussiness_type = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[3]/div[4]/div/p/text()').get()
        try:
            raw_bussiness_type = raw_bussiness_type.strip()
        except:
            raw_bussiness_type = 'ERRRRRRRRRRRRRRRRRRRORRRRRRRRRRRRRRRRRRRRRR:' + response.url.split('/')[-1]

        if raw_bussiness_type == '-'  or raw_bussiness_type == None:
            raw_bussiness_type = response.xpath('/html/body/div/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[3]/div[2]/div/p/text()').get().strip()

        tel = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[2]/table/tr[3]/td[2]/text()').get().strip()
        if tel == None:
            tel = '-'

        fax = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[2]/table/tr[4]/td[2]/text()').get().strip()
        if fax == None:
            fax = '-'

        website = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[2]/table/tr[5]/td[2]/text()').get().strip()
        if website == None:
            website = '-'

        email = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[2]/table/tr[6]/td[2]/text()').get().strip()
        if email == None:
            email = '-'

        item = PostscrapeItem()
        item['company_id']          = response.xpath('/html/body/div/div[4]/div[2]/div/div[2]/div[1]/div/div[1]/p/text()').get().strip()
        item['company_name']        = response.xpath('/html/body/div/div[4]/div[2]/div[1]/div[1]/h2/text()').get().strip()
        item['company_type']        = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[1]/table/tr[1]/th[2]/text()').get().strip()
        item['status']              = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[1]/table/tr[3]/td[2]/text()').get().strip()
        item['objective']           = objective.strip()
        item['directors']           = director_list
        item['bussiness_type']      = raw_bussiness_type
        item['address']             = response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[2]/table/tr[2]/td/text()').get().strip()
        item['tel']                 = tel
        item['fax']                 = fax
        item['website']             = website
        item['email']               = email

    #     item = []
    #     item.append({
    #         'company_name': response.xpath('/html/body/div/div[4]/div[2]/div[1]/div[1]/h2/text()').get(),
    #         'company_id': '0105554123553',
    #         'company_type': response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[1]/table/tr[1]/th[2]/text()').get(),
    #         'status': response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[1]/table/tr[3]/td[2]/text()').get(),
    #         'address': response.xpath('/html/body/div[1]/div[4]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/div[2]/table/tr[2]/td/text()').get(),
    #         'objective': objective,
    #         'directors': director_list,
    #         'bussiness_type': raw_bussiness_type,
    #         'tel': tel,
    #         'fax': fax,
    #         'website': website,
    #         'email': email,
    #     })

        print('------------Target Target Target------------')
        # self.writeJsonFile(item)
        # item = self.readLoadsFile()
        print(item)

        return item


from keras import Model
from keras.layers import BatchNormalization, Dense, Dropout

from tasks.model.config import get_model_configuration

model_config = get_model_configuration()

class Encoder(Model):
    def __init__(self):
        super(Encoder, self).__init__()
        self.D = Dense(
            units=model_config.EMBEDDING_DIM,
            activation="relu",
        )
        self.dropout = Dropout(model_config.DROPOUT_RATE)
        self.bna = BatchNormalization()

    def call(self, image_features, training=False):
        output = self.D(image_features)
        output = self.bna(output, training=training)
        if training:
            output = self.dropout(output)
        return output


from abc import ABC
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BertModel

RNN_TYPES = ['RNN', 'LSTM', 'GRU']

class AbstractModel(ABC):
    def predict_proba(self, batch):
        self.eval()
        y_pred = self.forward(batch)
        if self.output_dim == 1:
            y_pred = torch.sigmoid(y_pred)
            y_pred = torch.cat([1.-y_pred, y_pred], dim=1)
        else:
            y_pred = F.softmax(y_pred, dim=1)
        return y_pred

    def predict(self, batch):
        self.eval()
        y_pred = self.forward(batch)
        if self.output_dim == 1:
            y_pred = torch.sigmoid(y_pred)
            out = torch.as_tensor((y_pred - 0.5) > 0,
                                  dtype=torch.long, device=self.device)
        else:
            y_pred = F.softmax(y_pred, dim=1)
            out = torch.argmax(y_pred, dim=1)
        return out

class RNN(nn.Module, AbstractModel):
    def __init__(self, embedding_dim, hidden_dim, output_dim,
                 num_layers, pretrained_embeddings, bidirectional,
                 dropout_p=0., padding_idx=0, rnn_type='RNN',
                 device='cuda' if torch.cuda.is_available() else 'cpu'):

        super().__init__()
        self.output_dim = output_dim
        self.bidirectional = bidirectional
        self.device = device

        self.embedding = nn.Embedding.from_pretrained(pretrained_embeddings,
                                                      padding_idx=padding_idx)

        drop_prob = 0. if num_layers > 1 else dropout_p
        assert rnn_type in RNN_TYPES, f'Use one of the following: {str(RNN_TYPES)}'
        RnnCell = getattr(nn, rnn_type)
        self.rnn = RnnCell(embedding_dim,
                           hidden_dim,
                           num_layers=num_layers,
                           bidirectional=bidirectional,
                           dropout=drop_prob)

        self.dropout = nn.Dropout(dropout_p)
        self.fc = nn.Linear(hidden_dim * 2, output_dim)

    def forward(self, batch):
        # x_in: S x B
        x_in = batch.text

        # embedded: S x B x E
        embedded = self.dropout(self.embedding(x_in))

        # out: S x B x (H*num_directions)
        # hidden: (L*num_directions) x B x H
        out, hidden = self.rnn(embedded)
        if type(hidden) == tuple: hidden = hidden[0]

        # if bidirectional concat the final forward (hidden[-2,:,:]) and
        # backward (hidden[-1,:,:]) hidden layers, otherwise extract the
        # final hidden state and apply dropout
        # hidden = B x (H*num_directions)
        if self.bidirectional:
            hidden = torch.cat((hidden[-2, :, :], hidden[-1, :, :]), dim=1)
        else:
            hidden = hidden[-1, :, :]

        hidden = self.dropout(hidden)
        return self.fc(hidden)

class PackedRNN(RNN):
    def forward(self, batch):
        # x_in: S x B
        x_in, lengths = batch.text

        # embedded: S x B x E
        embedded = self.dropout(self.embedding(x_in))

        # pack sequence
        # output over padding tokens are zero tensors
        # hidden: (L*num_directions) x B x H
        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, lengths)
        packed_out, hidden = self.rnn(packed_embedded)
        if type(hidden) == tuple: hidden = hidden[0]

        # unpack sequence
        # out: S x B x (H*num_directions)
        out, out_lengths = nn.utils.rnn.pad_packed_sequence(packed_out)

        # if bidirectional concat the final forward (hidden[-2,:,:]) and
        # backward (hidden[-1,:,:]) hidden layers, otherwise extract the
        # final hidden state and apply dropout
        # hidden = B x (H*num_directions)
        if self.bidirectional:
            hidden = torch.cat((hidden[-2, :, :], hidden[-1, :, :]), dim=1)
        else:
            hidden = hidden[-1, :, :]

        hidden = self.dropout(hidden)
        return self.fc(hidden)

class BertClassifier(nn.Module, AbstractModel):
    def __init__(self, output_dim, bert_model_name='bert-base-uncased'):
        super().__init__()
        self.output_dim = output_dim
        self.bert = BertModel.from_pretrained(bert_model_name)
        config = self.bert.config
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.fc = nn.Linear(config.hidden_size, output_dim)
        nn.init.xavier_normal_(self.fc.weight)

    def forward(self, batch):
        x_in = batch.text
        _, pooled_output = self.bert(x_in.permute(1, 0))
        pooled_output = self.dropout(pooled_output)
        logits = self.fc(pooled_output)
        return logits

class RNN_NLI(nn.Module, AbstractModel):
    def __init__(self, embedding_dim, hidden_dim, output_dim,
                 num_layers, pretrained_embeddings, bidirectional,
                 dropout_p=0., padding_idx=0, rnn_type='LSTM',
                 num_out_layers=4,
                 device='cuda' if torch.cuda.is_available() else 'cpu'):

        super().__init__()
        self.output_dim = output_dim
        self.bidirectional = bidirectional
        self.device = device

        self.embedding = nn.Embedding.from_pretrained(pretrained_embeddings,
                                                      padding_idx=padding_idx)

        self.projection = nn.Linear(embedding_dim, embedding_dim)
        self.dropout = nn.Dropout(dropout_p)

        drop_prob = 0. if num_layers > 1 else dropout_p
        assert rnn_type in RNN_TYPES, f'Use one of the following: {str(RNN_TYPES)}'
        RnnCell = getattr(nn, rnn_type)
        self.rnn = RnnCell(embedding_dim,
                           hidden_dim,
                           num_layers=num_layers,
                           bidirectional=bidirectional,
                           dropout=drop_prob)

        self.relu = nn.ReLU()

        out_layers = []
        lin_dim = hidden_dim*2
        if bidirectional: lin_dim *= 2
        for _ in range(num_out_layers-1):
            out_layers.append(nn.Linear(lin_dim, lin_dim))
            out_layers.append(self.relu)
            out_layers.append(self.dropout)
        out_layers.append(nn.Linear(lin_dim, output_dim))
        self.out = nn.Sequential(*out_layers)

    def forward(self, batch):
        premise_embed = self.embedding(batch.premise)
        hypothesis_embed = self.embedding(batch.hypothesis)
        premise_proj = self.relu(self.projection(premise_embed))
        hypothesis_proj = self.relu(self.projection(hypothesis_embed))
        encoded_premise, _ = self.rnn(premise_proj)
        encoded_hypothesis, _ = self.rnn(hypothesis_proj)
        premise = encoded_premise.sum(dim=0)
        hypothesis = encoded_hypothesis.sum(dim=0)
        combined = torch.cat((premise, hypothesis), 1)
        return self.out(combined)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 30 2018

@author: dang

Written for Python 2.7
Test command:
python post_processing_table.py xlsFile csvFile new_xlsFile
"""
import sys
import os
import csv
import xlrd
import re
import Csv_Excel

text_to_move = [u'(các)',
                u'(thuộc)',
                u'(tính)',
                u'(chứng)',
                u'(có)',
                u'(sự)',
                u'(được)',
                u'(kỹ thuật)',
                u'(hiện tượng)',
                u'(trạng thái)',
                u'(nhóm)',
                u'(bị)',
                u'(tác nhân)',
                u'(quá trình)',
                u'(phương pháp)',
                u'(cơn)',
                u'(vật)',
                u'(một)',
                u'(thuyết)',
                u'(chất)',
                u'(độ)',
                u'(quang)',
                u'(cái)',
                u'(than)',
                u'(phụ)',
                u'(tờ)',
                u'(loại)',
                u'(mạch)',
                u'(phụ)',
                u'(những)'
                ]

def post_process_xls(xlsFile, csvFile, new_xlsFile, reportFile):
    work_book = xlrd.open_workbook(xlsFile)
    work_sheet = work_book.sheet_by_index(0)
    report = ''
    with open(csvFile, 'wb') as file_in:
        text_writer = csv.writer(file_in, delimiter='\t', quoting=csv.QUOTE_ALL)
        for rownum in range(work_sheet.nrows):
            text = work_sheet.row_values(rownum)
            # Create an empty column on the left of Vietnamese column
            text.append(text[4])
            text[4] = ''
            for word in text_to_move:
                if re.search(re.escape(word)+r'$', text[3]):
                    text[3] = re.sub(re.escape(word)+r'$', '', text[3]).strip()
                    text[4] = word+u' '+text[4]
            text_writer.writerow(
                list(x.encode('utf-8') if type(x) == type(u'') else x
                    for x in text))
            if rownum>0 and rownum<work_sheet.nrows-2:
                this_word = work_sheet.row_values(rownum)[3]
                prev_word = work_sheet.row_values(rownum-1)[3]
                next_word = work_sheet.row_values(rownum+1)[3]
                if len(this_word)>0 and len(prev_word)>0 and len(next_word)>0:
                    if this_word[0].upper()!=prev_word[0].upper() and prev_word[0].upper()==next_word[0].upper():
                        report += 'Check 1st letter in row ' + str(rownum) + '\n'
    Csv_Excel.csv_to_xls(csvFile, new_xlsFile)
    with open(reportFile, 'wt') as text_file:
        text_file.write(report)

source_dir = str(sys.argv[1])
if len(sys.argv) >= 3:
    result_dir = str(sys.argv[2])

if not os.path.exists(result_dir):
    os.makedirs(result_dir)

for (dirname, dirs, files) in os.walk(source_dir):
    for filename in files:
        filename_main, file_extension = os.path.splitext(filename)
        print(filename_main)
        print(file_extension)
        if file_extension == '.xls':
            xls_file = os.path.join(dirname,filename)
            csv_file = os.path.join(result_dir, filename_main + '.csv')
            new_xls_file = os.path.join(result_dir, filename_main + '.xls')
            log_file_path = os.path.join(result_dir, filename_main + '_post_process_stats.txt')
            post_process_xls(xls_file, csv_file, new_xls_file, log_file_path)


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

from tensorflow.keras import Model, layers
from tensorflow.keras.datasets import mnist

# MNIST dataset params
n_classes = 10
n_features = 784

# Training params
l_rate = 0.01
epoches = 1000
batch_size = 256
display_steps = 100

# Network param
n_hidden_1 = 128
n_hidden_2 = 256

(X_train, y_train), (X_test, y_test) = mnist.load_data()

# Convert to float32
X_train, X_test = np.array(X_train, np.float32), np.array(X_test, np.float32)

# Flatten images to 1-D array
X_train, X_test = X_train.reshape([-1, n_features]), X_test.reshape([-1, n_features])

# Normalize images values from [0, 255] to [0,1]
X_train, X_test = X_train / 255., X_test / 255.

# Shuffle data using tensorflow tf.data
train_data = tf.data.Dataset.from_tensor_slices((X_train, y_train))
train_data = train_data.repeat().shuffle(5000).batch(batch_size).prefetch(1)

# Create TF Model

class NeuralNet(Model):

    def __init__(self):
        super(NeuralNet, self).__init__()

        # First fully-connected hidden layer
        self.fc1 = layers.Dense(n_hidden_1, activation=tf.nn.relu)

        # Second fully connected hidden layer
        self.fc2 = layers.Dense(n_hidden_2, activation=tf.nn.relu)

        # Output layer
        self.out = layers.Dense(n_classes)

    # set forward pass
    def call(self, x, is_training=False):
        x = self.fc1(x)
        x = self.fc2(x)
        x = self.out(x)

        if not is_training:
            x = tf.nn.softmax(x)

        return x

# Build network model
neural_net = NeuralNet()

# Cross-entropy
def cross_entrpoy(x, y):
    y = tf.cast(y, tf.int64)
    # Apply softmax to logits compute cross-entropy
    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=x)
    return tf.reduce_mean(loss)

# Accuracy metric
def accuracy(y_pred, y_true):
    curr_pred = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64))

    return tf.reduce_mean(tf.cast(curr_pred, tf.float32), axis=-1)

# Stochastic gradient descent
optimizer = tf.optimizers.SGD(l_rate)

# Optimazation process
def run_optimazatio(x, y):
    with tf.GradientTape() as g:
        pred = neural_net(x, is_training=True)

        loss = cross_entrpoy(pred, y)

    # Variable update i.e trainable  variables
    trainable_vars = neural_net.trainable_variables

    # Compute gradiennts
    gradients = g.gradient(loss, trainable_vars)
    # Update W and b following gradients
    optimizer.apply_gradients(zip(gradients, trainable_vars))

# Run training for  the given steps
for step, (batch_x, batch_y) in enumerate(train_data.take(epoches), 1):
    run_optimazatio(batch_x, batch_y)

    if step % display_steps == 0:
        pred = neural_net(batch_x, is_training=True)
        loss = cross_entrpoy(pred, batch_y)

        acc = accuracy(pred, batch_y)
        print("step: %i, loss: %f, accuracy: %f" % (step, loss, acc))

pred = neural_net(X_test, is_training=False)
print("Accuracy: %f" % accuracy(pred, y_test))

n_images = 5
test_images = X_test[:n_images]

predictions = neural_net(test_images)

for i in range(n_images):
    plt.imshow(np.reshape(test_images[i], [28, 28]), cmap='gray')
    plt.show()

    print("Model prediction: %i" % np.argmax(predictions.numpy()[i]))


from __future__ import annotations

from typing import Any, Dict

from boa3.internal.model.builtin.interop.nativecontract import ContractManagement
from boa3.internal.model.builtin.native.inativecontractclass import INativeContractClass
from boa3.internal.model.method import Method

class ContractManagementClass(INativeContractClass):
    """
    A class used to represent ContractManagement native contract
    """

    def __init__(self):
        super().__init__('ContractManagement', ContractManagement)

    @property
    def class_methods(self) -> Dict[str, Method]:
        # avoid recursive import
        from boa3.internal.model.builtin.interop.interop import Interop

        if len(self._class_methods) == 0:
            from boa3.internal.model.builtin.native.contract_management import HasMethod

            self._class_methods = {
                'get_minimum_deployment_fee': Interop.GetMinimumDeploymentFee,
                'get_contract': Interop.GetContract,
                'has_method': HasMethod(),
                'deploy': Interop.CreateContract,
                'update': Interop.UpdateContract,
                'destroy': Interop.DestroyContract
            }
        return super().class_methods

    @classmethod
    def build(cls, value: Any = None) -> ContractManagementClass:
        if value is None or cls._is_type_of(value):
            return _ContractManagement

    @classmethod
    def _is_type_of(cls, value: Any):
        return isinstance(value, ContractManagementClass)

_ContractManagement = ContractManagementClass()


from __future__ import annotations

from typing import List, Callable, Any, Optional
from ..exceptions import InstantiationError

class EventDispatcher(object):
    """Class implementing an event dispatcher for custom events."""
    __slots__ = ['events']

    __instance: EventDispatcher = None

    @staticmethod
    def get_instance() -> EventDispatcher:
        """Get an instance of the Dispatcher.

        Returns:
            EventDispatcher: An instance of the class.

        Raises:
            InstantiationError: If two instances of the class are created.
        """
        if EventDispatcher.__instance is None:
            EventDispatcher()

        return EventDispatcher.__instance

    def __init__(self) -> None:
        if EventDispatcher.__instance is not None:
            raise InstantiationError("This class is a singleton!")
        else:
            self.events = {}

            EventDispatcher.__instance = self

    def get_subscribers(self, event: str) -> List[Callable]:
        """Return subscribers of a given event.

        Args:
            event (str): Name of the event.

        Returns:
            List[Callable]: Subscribed callbacks for the event.
        """
        return self.events[event]

    def register(self, event: str, obj: Any, method: Optional[str] = None) -> None:
        """Register a subscriber for a given event. Subscriber is composed of a class instance and
        an optional method name. If method name is not provided, by default the `run` method is searched.

        Args:
            event (str): Name of the event.
            obj (Any): instance of a class of a subscriber.
            method (Optional[str]): method name of the subscriber to run as callback. If method name is not provided,
                by default the `run` method is searched.

        Returns:
            None
        """
        if event not in self.events:
            self.events[event] = []

        self.get_subscribers(event).append(getattr(obj, 'run' if not method else method))

    def dispatch(self, event: str, **kwargs: Any) -> None:
        """Dispatch a given event.

        Args:
            event (str): Name of the event.
            kwargs (Any): Arguments to pass to the event callback.

        Returns:
            None
        """
        if event not in self.events:
            return

        for callback in self.get_subscribers(event):
            callback(**kwargs)

    def unregister(self, event: str) -> None:
        """Unregister all callbacks of a given event.

        Args:
            event (str): Name of the event.

        Returns:
            None
        """
        if event not in self.events:
            return

        del self.events[event]


import xarray as xr
import numpy as np
import os
from tqdm import tqdm
import shutil
import subprocess
import zarr

compressor = zarr.Blosc(cname='zstd', clevel=3, shuffle=2)
shutil.rmtree('tmp/ds_mask', ignore_errors=True)
shutil.rmtree('tmp/ds_mask_new', ignore_errors=True)
labels = os.listdir('ws_mask/amazonas')
arrays = {}
for label in tqdm(labels):
    ds = xr.open_zarr(f'ws_mask/amazonas/{label}')
    da = ds['mask'].compute()
    arrays[label] = da
array_dims = ['lat', 'lon']
concat_dim = 'label'
pix_deg = 1 / 1200
tolerance = 0.1 * pix_deg
vmin, vmax = {}, {}
for d in array_dims:
    vmin[d] = np.inf
    vmax[d] = -np.inf
for label in arrays:
    da = arrays[label]
    this_vmin, this_vmax = {}, {}
    for d in array_dims:
        vmin[d] = min(vmin[d], np.min(da[d]).values)
        vmax[d] = max(vmax[d], np.max(da[d]).values)
coord = {}
for d in array_dims:
    if arrays['0'][d].values[1] - arrays['0'][d].values[0] > 0: # increasing
        coord[d] = np.arange(vmin[d], vmax[d]+tolerance, pix_deg)
    else:
        coord[d] = np.arange(vmax[d], vmin[d]-tolerance, -pix_deg)
max_len = max([len(label) for label in list(arrays)])
for label in tqdm(arrays):
    da = arrays[label]
    a = np.zeros((1, coord['lat'].shape[0], coord['lon'].shape[0]), dtype=np.uint8)
    dlat = int(round((coord['lat'][0] - da.lat.values[0]) / pix_deg))
    dlon = int(round((da.lon.values[0] - coord['lon'][0]) / pix_deg))
    a[0, dlat:dlat+da.shape[0], dlon:dlon+da.shape[1]] = da.values
    da_reindex = xr.DataArray(a, coords=[[label], coord['lat'], coord['lon']], dims=['label', 'lat', 'lon'])
    ds = da_reindex.to_dataset(name='mask')
    if os.path.exists('tmp/ds_mask'):
        shutil.rmtree('tmp/ds_mask_new', ignore_errors=True)
        ds.to_zarr('tmp/ds_mask_new', encoding={'label': {'dtype': f'|S{max_len}'}, 'mask': {'compressor': compressor}})
        ds_mask = zarr.open('tmp/ds_mask', mode='a')
        ds_new = zarr.open('tmp/ds_mask_new', mode='r')
        ds_mask['mask'].append(ds_new['mask'])
        ds_mask['label'].append(ds_new['label'])
    else:
        ds.to_zarr('tmp/ds_mask', encoding={'label': {'dtype': f'|S{max_len}'}, 'mask': {'compressor': compressor}})
subprocess.check_call('gsutil -m cp -r tmp/ds_mask/ gs://pangeo-data/amazonas/'.split())


import tkinter as tk
from PIL import Image, ImageTk
import sys

class Home_page(tk.Frame) :
	def __init__(self, parent, app) :
		self.settings = app.settings
		self.app = app

		super().__init__(parent)
		self.grid(row=0, column=0, sticky="nsew")

		self.mainframe = tk.Frame(self, width =self.settings.width, height = self.settings.height, bg="gray70")
		self.mainframe.pack(fill="both", expand=True)

		self.create_logo()
		self.create_button()

	def quit_game(self) :
		sys.exit()

	def create_button(self) :
		self.B_start = tk.Button(self.mainframe, text="Start",font = ("Arial", 16, "bold"), command=lambda:self.app.change_page("Game_page"), bg="gray60", bd=1, fg="white", width=15,height=1)
		self.B_start.pack(pady=10)

		self.B_htp = tk.Button(self.mainframe, text="How to Play",font = ("Arial", 16, "bold"), bg="gray60", bd=1, fg="white",height=1, width=15, command=self.htp)
		self.B_htp.pack(pady=10)

		self.B_leaderboard = tk.Button(self.mainframe, text="Leaderboard",font = ("Arial", 16, "bold"), bg="gray60", bd=1, fg="white",height=1, width=15, command=self.leaderboard)
		self.B_leaderboard.pack(pady=10)

		self.B_quit = tk.Button(self.mainframe, text="Quit",font = ("Arial", 16, "bold"), bg="gray60", bd=1, fg="white",height=1, width=15, command=self.quit_game)
		self.B_quit.pack(pady=10)

	def create_logo(self) :
		self.logo = ImageTk.PhotoImage(Image.open(self.settings.logo_path))
		self.L_logo = tk.Label(self.mainframe, image=self.logo, bg="gray70")
		self.L_logo.pack(pady=30)

	def htp(self) :
		self.app.change_page("htp1")

	def leaderboard(self) :
		self.app.change_page("Leaderboard")


import torch
import torch.nn as nn
from .UNet_torch import UNet, CropNConcat

def sinusoidal_embedding(n, d):
    """
    n: iteration steps,
    d: time embedding dimension
    """
    # Returns the standard positional embedding
    embedding = torch.tensor([[i / 10_000 ** (2 * j / d) for j in range(d)] for i in range(n)])
    sin_mask = torch.arange(0, n, 2)

    embedding[sin_mask] = torch.sin(embedding[sin_mask])
    embedding[1 - sin_mask] = torch.cos(embedding[sin_mask])

    return embedding

def _make_te(dim_in, dim_out):
        return nn.Sequential(
            nn.Linear(dim_in, dim_out),
            nn.SiLU(),
            nn.Linear(dim_out, dim_out)
        )

class UNet_DDPM(UNet):
    def __init__(self, input_shape, root_filter=64, layer_depth=5, drop_rate=0.2, n_steps=1000, time_emb_dim=100, training=True):
        super().__init__(input_shape, root_filter, layer_depth, drop_rate, training)

        ## sinusoidal embedding
        self.time_embed = nn.Embedding(n_steps, time_emb_dim)
        self.time_embed.weight.data = sinusoidal_embedding(n_steps, time_emb_dim)
        self.time_embed.requires_grad_(False)

        self.time_list = nn.ModuleList()
        for i in range(layer_depth-1):
            self.time_list.append(_make_te(time_emb_dim, self.filter_list[i]))

        self.time_list2 = nn.ModuleList()
        for i in range(layer_depth, 1, -1):
            self.time_list2.append(_make_te(time_emb_dim, self.filter_list[i]))

    def forward(self, x, t):
        skip_connects = []
        t = self.time_embed(t)
        nn = len(x)

        for dd, te in zip(self.downlist, self.time_list):
            ## add time
            x = x + te(t).reshape(nn, -1, 1, 1)
            x = dd(x)
            skip_connects.append(x)
            x = self.pooling2d(x)

        for index in range(len(self.uplist)):
            x = self.uplist[index](x)
            x = CropNConcat(x, skip_connects[len(skip_connects)-1-index])
            x = x + self.time_list2[index](t).reshape(nn, -1, 1, 1)
            x = self.downlist2[index](x)

        out = self.out_conv(x)

        return out

class DDPM(nn.Module):
    def __init__(self, backbone, input_shape, n_steps=1000, min_beta=10**(-4), max_beta=0.02):
        super(DDPM, self).__init__()

        self.backbone = backbone
        self.n_steps = n_steps
        self.input_shape = input_shape
        # Number of steps is typically in the order of thousands
        self.betas = torch.linspace(min_beta, max_beta, n_steps)

        self.alphas = 1 - self.betas
        self.alpha_bars = torch.tensor([torch.prod(self.alphas[:i + 1]) for i in range(len(self.alphas))])

    def forward(self, x0, t, eta=None):
        n, c, h, w = x0.shape
        a_bar = self.alpha_bars[t]

        if eta is None:
            eta = torch.randn(n, c, h, w)

        a_bar = a_bar.to(x0.device)

        noisy_x = a_bar.sqrt().reshape(n, 1, 1, 1) * x0 + (1 - a_bar).sqrt().reshape(n, 1, 1, 1) * eta

        return noisy_x

    def backward(self, x, t):

        x = x.to('cuda')
        t = t.to(x.device)

        return self.backbone(x, t)

if __name__ == '__main__':
    from torchinfo import summary
    from torch.utils.tensorboard import SummaryWriter
    import torch

    input_shape = (1, 28, 28)
    n_steps = 1000
    backbone = UNet_DDPM(input_shape=input_shape, n_steps=n_steps, layer_depth=4, drop_rate=0.2)
    ddpm_model = DDPM(backbone, input_shape, n_steps=n_steps)

    percent = 1
    t = [int(percent * (200-1)) for _ in range(16)]

    ##summary(model, input_size=(16, 4, 256, 256))
    #t = torch.randint(0, 1000, (16,))
    fake_input = torch.rand((16, 1, 28, 28))
    values = ddpm_model(fake_input, t)
    #values = backbone(fake_input, t=t)
    print('input:', fake_input.shape)
    print('output:', values.shape)
    #writer = SummaryWriter()
    #writer.add_graph(model, fake_input)
    #writer.close()


from django.shortcuts import render

# Create your views here.

#these are for API views, so we can try to udnerstand what's going on here
#imports:
from django.http import HttpResponse, JsonResponse
from django.views.decorators.csrf import csrf_exempt
from rest_framework.renderers import JSONRenderer
from rest_framework.parsers import JSONParser
from snippets.models import *
from snippets.serializers import *
from rest_framework.response import Response
from rest_framework.decorators import api_view
from rest_framework import status

# one simple way to refactor is to use the REST apis response instead of the JSON response

@api_view(['GET', 'POST']) #we use the api view annotator as it's a function view
def snippet_list(request, format=None):
	#list all snippets or create a new snippe
	if request.method == 'GET':
		snippets = Snippet.objects.all() #get all our snippets
		serializer = SnippetSerializer(snippets, many=True)
		return Response(serializer.data)
		#this actually works and makes a whole load of sense. wow. that's not too hard at all!

	elif request.method == 'POST':
		serializer = SnippetSerializer(data=request.data)
		if serializer.is_valid():
			serializer.save()
			return Response(serializer.data, status=status.HTTP_201_CREATED)
		return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

	#wow, that's not complicated at all. pretty easy and simple and nice yay!

#okay, this is a view corresponding to an individual snippet which can be used to retrieve, update or delete the snippet, I'm not sure how this is meant to work in an API but okay!

#the other benefit is we're no longer tied to json. Response can handle other formats, such as XML or whatever

# we can also add format views/suffixes which allow us to handle urls which explicitly refer to a given format i.e. api/items/4.json

@api_view(['GET', 'PUT', 'DELETE'])
def snippet_detail(request, pk, format=None):
	try:
		snippet = Snippet.objects.get(pk=pk)
	except Snippet.DoesNotExist:
		return Response(status=status.HTTP_404_NOT_FOUND)

	#okay, we gget the actual thing
	#now we begin the request properly
	if request.method=='GET':
		serializer =SnippetSerializer(snippet)
		return Response(serializer.data)

	elif request.method=='PUT':
		serializer = SnippetSerializer(snippet,data=request.data)
		if serializer.is_valid():
			serializer.save()
			return Response(serializer.data)
		return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

	elif request.method=='DELETE':
		snippet.delete()
		return Response(status=status.HTTP_204_NO_CONTENT)


import asyncio
import pulsectl
import logging
import coloredlogs
import re
import time

"""
    A simple Proxy between MPD and it's clients that intercepts the setvol command and uses
    that to control a pulseaudio sources's volume instead.

    Melan, 2020
"""

class mpdprox():
    mpd_server = "localhost"
    mpd_port = 6605
    sinkname = "alsa_output.platform-soc_sound.stereo-fallback"

    log = logging.getLogger("MpdVolumeProxy")

    def __init__(self, loop):
        self.loop = loop
        self.pulse = pulsectl.Pulse()
        self.log.info(f"Primary sink: {self.get_primary_sink()} ({self.get_volume()})")

    def start(self):
        self.server_coro = asyncio.start_server(self.handle_client, '0.0.0.0', 6600, loop=loop)
        self.server = loop.run_until_complete(self.server_coro )
        self.log.info('Serving on {}'.format(self.server.sockets[0].getsockname()))

        try:
            loop.run_forever()

        except KeyboardInterrupt:
            self.log.error("Got keyboard interrupt")

        finally:
            # Close the server
            self.server.close()
            self.loop.run_until_complete(self.server.wait_closed())
            self.loop.close()

    def get_primary_sink(self):
        for sink in self.pulse.sink_list():
            if sink.name == self.sinkname:
                return sink
        self.log.error(f"Failed to find sink named {self.sinkname}")

    def get_volume(self):
        sink = self.get_primary_sink()
        if sink:
            return round(self.pulse.volume_get_all_chans(sink) * 100)
        return 0

    def change_volume(self, procent):
        sink = self.get_primary_sink()
        if sink:
            current_volume = (round(self.pulse.volume_get_all_chans(sink) * 100))

            #Don't actually change the volume above 100
            if procent > 100:
                procent = 100

            if procent < 0:
                procent = 0

            change_to = procent - current_volume
            self.log.info(f"Changing volume to: {change_to}")
            self.pulse.volume_change_all_chans(sink, change_to / 100)

    def extract_numbers_str(self, str):
        return  re.findall(r'\d+', str)

    async def pipe(self, reader, writer, client, intercept=False, intercept_type=0):
        try:
            while not reader.at_eof():
                data = await reader.read(2048)
                if intercept:
                    # Intercept any data from client to mpd
                    if intercept_type == 0:
                        # if not data in [b"idle\n", b"noidle\n", b"status\n"]:
                        #     print(data)
                        # When the client tries to set the volume
                        if data.startswith(b"vol") or data.startswith(b"setvol"):
                            self.log.info(f"Intercepting VOL command: {data.decode('utf-8').strip()}")
                            if len(data.decode("utf-8").split(" ")) > 1:
                                volume = int(self.extract_numbers_str(data.decode("utf-8"))[0])
                                self.log.info(f"Setting pulse volume to {volume}")
                                self.change_volume(volume)
                                client.write(b"OK\n")
                            else:
                                client.write(b"Kut\n")

                        else:
                            writer.write(data)

                    # Intercept any data from mpd to client
                    elif intercept_type == 1:
                        # Intercept the volume flag and replace it with our own
                        if data.startswith(b"volume: "):
                            data = data.decode("utf-8")
                            data = "\n".join(data.split("\n")[1:])
                            data = "volume: " + str(self.get_volume()) + "\n" + data
                            writer.write(data.encode("utf-8"))
                        else:
                            writer.write(data)
                else:
                    writer.write(data)
        finally:
            writer.close()
            client.close()

    async def handle_client(self, local_reader, local_writer):
        """
            Handle when we get a new client, make a connection to the source MPD server
            and start relaying data between them. And intercept the volume command.s
        """
        info = local_writer.get_extra_info('socket')
        self.log.info(f"New client: {str(info)} ")
        try:
            remote_reader, remote_writer = await asyncio.open_connection(self.mpd_server, self.mpd_port, loop=self.loop)
            pipe1 = self.pipe(local_reader, remote_writer, local_writer, intercept=True, intercept_type=0)
            pipe2 = self.pipe(remote_reader, local_writer, local_writer, intercept=True, intercept_type=1)
            await asyncio.gather(pipe1, pipe2)
        except Exception as e:
            self.log.error(f"Exception: {e}")
        finally:
            local_writer.close()

loop = asyncio.get_event_loop()
logging.basicConfig(level=logging.INFO)
coloredlogs.install(level='INFO', fmt="%(asctime)s %(name)s %(levelname)s %(message)s")
while True:
    try:
        mp = mpdprox(loop)
        mp.start()
    except Exception as e:
        print(f"Exception! {e}")
        time.sleep(10)