#!/usr/bin/python3
import sys

from datetime import datetime
from chapter2 import Node

class Stack:
    def __init__(self):
        self.top = None
        self.front = None

    def push(self, data):
        new_node = Node(data)
        if self.top:
            new_node.next = self.top
        else:
            self.front = new_node
        self.top = new_node

    def pop(self):
        if self.top:
            popped = self.top.data
            self.top = self.top.next
            return popped
        return None

    def peep(self):
        if self.top:
            return self.top.data

    def isEmpty(self):
        if not self.top:
            return True
        else:
            return False

    def popFront(self):
        node = self.top
        if not node.next:
            return self.pop()
        while node.next:
            next = node.next
            if not next.next:
                popped = next.data
                node.next = None
            else:
                node = next
        return popped

    def print_all(self, prepend):
        node = self.top
        s = ''
        while node:
            s += f'{node.data} -> '
            node = node.next
        print(f'{prepend} {s[:-4]}')

class StackWithMin(Stack):
    def __init__(self):
        super().__init__()
        self.internal_stack = Stack()

    def push(self, data):
        if data < self.min():
            self.internal_stack.push(data)
        return super().push(data)

    def pop(self):
        data = super().pop()
        if data == self.min():
            self.internal_stack.pop()
        return data

    def peek(self):
        return super().peep()

    def min(self):
        stack_min = self.internal_stack.peep()
        if not stack_min:
            return sys.maxsize
        return self.internal_stack.peep()

def problem1() -> Stack:
    """How would you design a stack which, in addition to push and pop, also has a min
    function which returns the minimum element? Push, pop, and min should all operate on O(1) time"""
    st = StackWithMin()
    st.push(2)
    st.push(4)
    st.push(1)
    st.push(6)
    print(st.peep())
    print(f'm: {st.min()}')
    print(st.pop())
    print(st.pop())
    print(f'm: {st.min()}')
    print(st.peep())
    return st

class SetOfStacks:
    def __init__(self, maximum):
        self.current_length = 0
        self.maximum = maximum
        self.stacks = [Stack()]

    def push(self, data):
        if self.current_length == self.maximum:
            print('adding stack!')
            self.current_length = 0
            self.stacks.append(Stack())
        else:
            self.current_length += 1
        self.current_stack().push(data)

    def pop(self):
        popped = self.current_stack().pop()
        if self.current_length == 0:
            print('removing stack!')
            self.stacks.pop()
            self.current_length = self.maximum
        else:
            self.current_length -= 1

        return popped

    def peep(self):
        return self.current_stack().peep()

    def current_stack(self):
        return self.stacks[-1]

    def popAt(self, substack):
        stack = self.stacks[substack]
        popped = stack.pop()
        for i in range(substack, len(self.stacks) - 1):
            cur_stack = self.stacks[i]
            next_stack = self.stacks[i+1]
            front = next_stack.popFront()
            print(f'front: {front}')
            cur_stack.push(front)
        if self.current_length == 0:
            print("shouldnt be here")
            self.stacks.pop()
            self.current_length = self.maximum
        self.current_length -= 1

        return popped

def problem2():
    """Implement a data structure SetOfStacks that is composed of several stacks and should
    create a new stack once the previosu one exceeds capacity."""
    sos = SetOfStacks(2)
    sos.push(2)
    print(f'peep: {sos.peep()}')
    sos.push(4)
    print(f'peep: {sos.peep()}')
    sos.push(6)
    print(f'peep: {sos.peep()}')
    sos.push(8)
    print(f'peep: {sos.peep()}')
    print(f'special: {sos.popAt(0)}')
    print(f'p: {sos.pop()}')
    print(f'p: {sos.pop()}')
    print(f'p: {sos.pop()}')

class TowerOfHanoi:
    def __init__(self, depth):
        self.depth = depth
        self.stacks = {
            'left': Stack(),
            'mid': Stack(),
            'right': Stack()
        }
        for i in reversed(range(depth)):
            self.stacks['left'].push(i + 1)

    def find_stack(self, n):
        for stack in self.stacks.values():
            if stack.peep() == n:
                return stack

    def get_dir_from_stack(self, match):
        for dir, stack in self.stacks.items():
            if stack == match:
                return dir

    def print_stacks(self):
        print('----------------------------')
        for t, stack in self.stacks.items():
            prepend = f'{t} stack:'
            stack.print_all(prepend)
        print('----------------------------')

    def solve_tower(self, depth, source, buffer, destination):
        dir1, dir2, dir3 = self.get_dir_from_stack(source), self.get_dir_from_stack(buffer), self.get_dir_from_stack(destination)
        print(f'solver! depth: {depth}, source = {dir1}, buffer = {dir2}, dest = {dir3}')

        if not depth:
            return
        if depth == 1:
            destination.push(source.pop())
            return

        self.solve_tower(depth-1, source, destination, buffer)
        self.print_stacks()

        destination.push(source.pop())
        self.print_stacks()

        self.solve_tower(depth-1, buffer, source, destination)
        self.print_stacks()

    def visualize_tower(self):
        list_of_values = []
        for stack in self.stacks.values():
            vals = []
            while stack.peep():
                vals.append(stack.pop())
            reversed(vals)

            list_of_values.append(vals)

        index = 0
        for stack in self.stacks.values():
            cur_list = list_of_values[index]
            for item in cur_list:
                stack.push(item)
            index += 1

        for l in list_of_values:
            for i in range(self.depth - len(l)):
                l.append('|')

        for i in range(len(list_of_values)):
            list_of_values[i] = list(reversed(list_of_values[i]))

        columns = []
        for i in range(self.depth):
            columns.append([])

        for l in list_of_values:
            for i in reversed(range(len(l))):
                columns[i].append(str(l[i]))

        for col in columns:
            print(' '.join(col))
        print("_ _ _")

        return list_of_values

def problem3(n: int):
    """towers of hanoi"""

    towers = TowerOfHanoi(n)
    towers.print_stacks()
    towers.solve_tower(n, towers.stacks['left'], towers.stacks['mid'], towers.stacks['right'])

class MyQueue:
    def __init__(self):
        self.older = Stack()
        self.newer = Stack()
        self.length = 0

    def enqueue(self, data):
        if self.older.peep():
            popped = self.older.pop()
            while popped:
                self.newer.push(popped)
                popped = self.older.pop()
        self.length += 1

        self.newer.push(data)

    def dequeue(self):
        popped = self.newer.pop()
        while popped:
            self.older.push(popped)
            popped = self.newer.pop()
        self.length -= 1
        return self.older.pop()

    def peek(self):
        popped = self.newer.pop()
        while popped:
            self.older.push(popped)
            popped = self.newer.pop()
        return self.older.peep()

    def empty(self):
        return not self.length

def problem4():
    """Implement a MyQueue class which implements a queue using two stacks"""
    q = MyQueue()
    q.enqueue(1)
    q.enqueue(2)
    q.enqueue(3)
    print(q.dequeue())
    print(q.dequeue())
    q.enqueue(11)
    q.enqueue(12)
    print(q.dequeue())
    print(q.dequeue())
    print(q.dequeue())

def problem5():
    """Write a program to sort a stack in ascending order (with biggest items on top). You may
    use at most one additional stack to hold items, but you may not copy the elements into any
    other data structure. The stack supports push, pop, peek, and isEmpty()"""
    stack = Stack()
    stack.push(8)
    stack.push(1)
    stack.push(6)
    stack.push(3)
    stack.push(4)
    stack.push(2)
    stack.push(5)

    sorted_stack = Stack()

    while not stack.isEmpty():
        stack.print_all('starting: ')
        sorted_stack.print_all('sorted: ')

        value_to_sort = stack.pop()
        print(f'value to sort: {value_to_sort}')
        if sorted_stack.isEmpty():
            sorted_stack.push(value_to_sort)
        else:
            while not sorted_stack.isEmpty():
                if value_to_sort < sorted_stack.peep():
                    print('here')
                    sorted_stack.push(value_to_sort)
                    break
                else:
                    stack.push(sorted_stack.pop())
    return sorted_stack

class Animal:
    def __init__(self, name):
        self.time_created = datetime.now()
        self.name = name

    def is_older(self, other: 'Animal'):
        return self.time_created < other.time_created

class Dog(Animal):
    def __init__(self, name):
        super().__init__(name)

class Cat(Animal):
    def __init__(self, name):
        super().__init__(name)
        self.name = name

    def get_name(self):
        return self.name

class AnimalQueue:
    def __init__(self):
        self.cat_queue = MyQueue()
        self.dog_queue = MyQueue()
        self.oldest_cat = None
        self.oldest_dog = None

    def isEmpty(self):
        return self.dog_queue.empty() and self.cat_queue.empty()

    def enqueue(self, animal: Animal):
        if isinstance(animal, Cat):
            self.cat_queue.enqueue(animal)
        else:
            self.dog_queue.enqueue(animal)

    def dequeue_dog(self):
        if self.dog_queue.empty():
            return None
        return self.dog_queue.dequeue()

    def dequeue_cat(self):
        if self.cat_queue.empty():
            return None
        return self.cat_queue.dequeue()

    def dequeue_any(self):
        dog = self.dog_queue.peek()
        cat = self.cat_queue.peek()
        if not dog or cat.is_older(dog):
            return self.dequeue_cat()
        if not cat or dog.is_older(cat):
            return self.dequeue_dog()

def problem6():
    """An animal shelter holds only dogs and cats, and operations on a strictly first in first out
    basis. People can select where they want a cat or a dog, or have no preference. Create a data
    structure to maintain this system and implement operations such as enqueue, dequeueAny, dequeueDog, dequeueCat"""

    q = AnimalQueue()
    q.enqueue(Dog('Fritz'))
    q.enqueue(Cat('Frida'))
    q.enqueue(Cat('Diego'))
    q.enqueue(Dog('Frankie'))

    print(q.dequeue_cat().name)
    print(q.dequeue_cat().name)
    print(q.dequeue_dog().name)
    print(q.dequeue_dog().name)
    q.enqueue(Dog('Chorizo'))
    q.enqueue(Cat('Kahlo'))
    print(q.dequeue_any().name)
    print(q.dequeue_any().name)

if __name__ == "__main__":
    problem1()
    problem2()
    problem3(5)
    problem4()
    problem5()
    problem6()


#
# Tests for training and evaluation of all models.
#

import logging
import os
import pytest
import numpy as np
from datetime import datetime
from src.models.data_manager import DataManager
from src.models import train_model
from src.models.predict_model import predict_seqtoseq, predict_seqtoseq_beam
from src.models.agent import PolicyAgent
from src.models.policy_model import PolicyGradientModel

#
# Configure logging
#
abs_path = os.path.abspath(os.path.dirname(__file__))
logs_dir = os.path.join(abs_path, '../../logs')
if not os.path.exists(logs_dir):
    os.mkdir(logs_dir)
    os.chmod(logs_dir, 0o777)
log_path = os.path.join(abs_path, '../../logs/run-{0}.log')
logging.basicConfig(filename=log_path.format(datetime.now().strftime('%Y%m%d-%H%M%S')),
                    level=logging.INFO,
                    format='%(asctime)s-%(process)d-%(name)s-%(levelname)s-%(message)s',
                    datefmt='%Y-%m-%d %H:%M:%S')

logging.info('Test Logger Created.')

class TestModel(object):

    def test_get_cornell_data(self):
        d = DataManager()
        x, y, a, b = d.get_cornell_data()
        assert x is not None
        assert y is not None
        assert a is not None
        assert b is not None

    def test_get_starting_prompts(self):
        questions = DataManager().get_cornell_starting_prompts()
        print(questions)
        assert len(questions) > 0

    def test_get_dull_responses(self):
        questions = DataManager().get_cornell_dull_responses()
        print(questions)
        assert len(questions) > 0

    def test_train_seqtoseq(self):
        train_model.train_seqtoseq(model_name='test-seqtoseq')

    def test_predict_seqtoseq(self):
        y = predict_seqtoseq('Hi!', model_name='test-seqtoseq')
        print(y)

    def test_train_policy(self):
        train_model.train_rl(model_name='test-rl')

    def test_predict_policy(self):
        y = predict_seqtoseq_beam('Hi!', model_name='test-seqtoseq-attn')
        print(y)

    def test_policy_reward(self):
        data_manager = DataManager()
        agent = PolicyAgent(agent_name='test-agent')
        last_response = data_manager.question_to_tokens('I am fine. How are you?')
        request = data_manager.question_to_tokens('Great. How is the weather today?')
        responses, probs, rewards = agent.play((last_response, request))
        logging.info(responses)
        logging.info(probs)
        logging.info(rewards)
        assert len(responses) > 0

    @pytest.mark.run_this
    def test_rl_train(self):
        batch_size = 1100
        model = PolicyGradientModel(seq2seq_model_name='test-policy', model_name='rl-policy')
        model_inputs = {
            'request': [[np.random.randint(0, 5000) for _ in range(10)]] * batch_size,
            'response': [[np.random.randint(0, 5000) for _ in range(15)]] * batch_size,
            'probability': [[np.random.random() for _ in range(15)]] * batch_size,
            'reward': [[np.random.random() for _ in range(15)]] * batch_size
        }

        model.train(model_inputs, None)


import logging
import os
import random
import uuid
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Dict, List, Optional, Tuple

import great_expectations as gx
from delta.tables import DeltaTable
from faker import Faker
from pyspark.sql import DataFrame, SparkSession
from pyspark.sql.functions import col, current_timestamp, expr, lit
from pyspark.sql.types import (
    IntegerType,
    StringType,
    StructField,
    StructType,
    TimestampType,
)

########################################################################
# GENERATING FAKE BRONZE DATA !!!
########################################################################

def _get_orders(
    cust_ids: List[int], num_orders: int
) -> List[Tuple[str, int, str, str, datetime, datetime]]:
    items = [
        "chair",
        "car",
        "toy",
    ]
    return [
        (
            str(uuid.uuid4()),
            int(random.choice(cust_ids)),
            str(uuid.uuid4()),
            random.choice(items),
            datetime.now(),
            datetime.now(),
        )
        for _ in range(num_orders)
    ]

def _get_customer_data(
    cust_ids: List[int],
) -> List[Tuple[int, str, str, str, datetime, datetime]]:
    fake = Faker()
    return [
        (
            cust_id,
            fake.first_name(),
            fake.last_name(),
            fake.state_abbr(),
            datetime.now(),
            datetime.now(),
        )
        for cust_id in cust_ids
    ]

def generate_bronze_data(
    spark: SparkSession,
    iteration: int = 1,
    orders_bucket: str = "app-orders",
    **kwargs,
) -> List[DataFrame]:
    cust_ids = [i for i in range(1000)]
    return [
        spark.createDataFrame(
            data=_get_customer_data(cust_ids),
            schema=StructType(
                [
                    StructField("id", IntegerType(), True),
                    StructField("first_name", StringType(), True),
                    StructField("last_name", StringType(), True),
                    StructField("state_id", StringType(), True),
                    StructField("datetime_created", TimestampType(), True),
                    StructField("datetime_updated", TimestampType(), True),
                ]
            ),
        ),
        spark.createDataFrame(
            data=_get_orders(cust_ids, 10000),
            schema=StructType(
                [
                    StructField("order_id", StringType(), True),
                    StructField("customer_id", IntegerType(), True),
                    StructField("item_id", StringType(), True),
                    StructField("item_name", StringType(), True),
                    StructField("delivered_on", TimestampType(), True),
                    StructField(
                        "datetime_order_placed", TimestampType(), True
                    ),
                ]
            ),
        ),
    ]

########################################################################

@dataclass
class DeltaDataSet:
    name: str
    curr_data: DataFrame
    primary_keys: List[str]
    storage_path: str
    table_name: str
    data_type: str
    database: str
    partition: str
    skip_publish: bool = False
    replace_partition: bool = False

class InValidDataException(Exception):
    pass

class StandardETL(ABC):
    def __init__(
        self,
        storage_path: Optional[str] = None,
        database: Optional[str] = None,
        partition: Optional[str] = None,
    ):
        self.STORAGE_PATH = storage_path or 's3a://adventureworks/delta'
        self.DATABASE = database or 'adventureworks'
        self.DEFAULT_PARTITION = partition or datetime.now().strftime(
            "%Y-%m-%d-%H-%M-%S"
        )

    def run_data_validations(self, input_datasets: Dict[str, DeltaDataSet]):
        context = gx.get_context(
            context_root_dir=os.path.join(
                os.getcwd(),
                "adventureworks",
                "great_expectations",
            )
        )

        validations = []
        for input_dataset in input_datasets.values():
            validations.append(
                {
                    "batch_request": context.get_datasource("spark_datasource")
                    .get_asset(input_dataset.name)
                    .build_batch_request(dataframe=input_dataset.curr_data),
                    "expectation_suite_name": input_dataset.name,
                }
            )
        return context.run_checkpoint(
            checkpoint_name="dq_checkpoint", validations=validations
        ).list_validation_results()

    def validate_data(self, input_datasets: Dict[str, DeltaDataSet]) -> bool:
        results = {}
        for validation in self.run_data_validations(input_datasets):
            results[
                validation.get('meta').get('expectation_suite_name')
            ] = validation.get('success')
        for k, v in results.items():
            if not v:
                raise InValidDataException(
                    f"The {k} dataset did not pass validation, please check"
                    " the metadata db for more information"
                )

        return True

    def check_required_inputs(
        self, input_datasets: Dict[str, DeltaDataSet], required_ds: List[str]
    ) -> None:
        if not all([ds in input_datasets for ds in required_ds]):
            raise ValueError(
                f"The input_datasets {input_datasets.keys()} does not contain"
                f" {required_ds}"
            )

    def construct_join_string(self, keys: List[str]) -> str:
        return ' AND '.join([f"target.{key} = source.{key}" for key in keys])

    def publish_data(
        self,
        input_datasets: Dict[str, DeltaDataSet],
        spark: SparkSession,
        **kwargs,
    ) -> None:
        for input_dataset in input_datasets.values():
            if not input_dataset.skip_publish:
                curr_data = input_dataset.curr_data.withColumn(
                    'etl_inserted', current_timestamp()
                ).withColumn('partition', lit(input_dataset.partition))
                if input_dataset.replace_partition:
                    curr_data.write.format("delta").mode("overwrite").option(
                        "replaceWhere",
                        f"partition = '{input_dataset.partition}'",
                    ).save(input_dataset.storage_path)
                else:
                    targetDF = DeltaTable.forPath(
                        spark, input_dataset.storage_path
                    )
                    (
                        targetDF.alias("target")
                        .merge(
                            curr_data.alias("source"),
                            self.construct_join_string(
                                input_dataset.primary_keys
                            ),
                        )
                        .whenMatchedUpdateAll()
                        .whenNotMatchedInsertAll()
                        .execute()
                    )

    @abstractmethod
    def get_bronze_datasets(
        self, spark: SparkSession, **kwargs
    ) -> Dict[str, DeltaDataSet]:
        pass

    @abstractmethod
    def get_silver_datasets(
        self,
        input_datasets: Dict[str, DeltaDataSet],
        spark: SparkSession,
        **kwargs,
    ) -> Dict[str, DeltaDataSet]:
        pass

    @abstractmethod
    def get_gold_datasets(
        self,
        input_datasets: Dict[str, DeltaDataSet],
        spark: SparkSession,
        **kwargs,
    ) -> Dict[str, DeltaDataSet]:
        pass

    def run(self, spark: SparkSession, **kwargs):
        partition = kwargs.get('partition')
        bronze_data_sets = self.get_bronze_datasets(spark, partition=partition)
        self.validate_data(bronze_data_sets)
        self.publish_data(bronze_data_sets, spark)
        logging.info(
            'Created, validated & published bronze datasets:'
            f' {[ds for ds in bronze_data_sets.keys()]}'
        )

        silver_data_sets = self.get_silver_datasets(
            bronze_data_sets, spark, partition=partition
        )
        self.validate_data(silver_data_sets)
        self.publish_data(silver_data_sets, spark)
        logging.info(
            'Created, validated & published silver datasets:'
            f' {[ds for ds in silver_data_sets.keys()]}'
        )

        gold_data_sets = self.get_gold_datasets(
            silver_data_sets, spark, partition=partition
        )
        self.validate_data(gold_data_sets)
        self.publish_data(gold_data_sets, spark)
        logging.info(
            'Created, validated & published gold datasets:'
            f' {[ds for ds in gold_data_sets.keys()]}'
        )

class SalesMartETL(StandardETL):
    def get_bronze_datasets(
        self, spark: SparkSession, **kwargs
    ) -> Dict[str, DeltaDataSet]:
        customer_df, orders_df = generate_bronze_data(spark)
        return {
            'customer': DeltaDataSet(
                name='customer',
                curr_data=customer_df,
                primary_keys=['id', 'partition'],
                storage_path=f'{self.STORAGE_PATH}/customer',
                table_name='customer',
                data_type='delta',
                database=f'{self.DATABASE}',
                partition=kwargs.get('partition', self.DEFAULT_PARTITION),
                replace_partition=True,
            ),
            'orders': DeltaDataSet(
                name='orders',
                curr_data=orders_df,
                primary_keys=['order_id', 'partition'],
                storage_path=f'{self.STORAGE_PATH}/orders',
                table_name='orders',
                data_type='delta',
                database=f'{self.DATABASE}',
                partition=kwargs.get('partition', self.DEFAULT_PARTITION),
                replace_partition=True,
            ),
        }

    def get_dim_customer(
        self, customer: DeltaDataSet, spark: SparkSession, **kwargs
    ) -> DataFrame:
        customer_df = customer.curr_data
        dim_customer = kwargs['dim_customer']
        # generete pk
        customer_df = customer_df.withColumn(
            'customer_sur_id',
            expr('md5(concat(id, datetime_updated))'),
        )
        # get only latest customer rows in dim_customer
        # since dim customer may have multiple rows per customer (SCD2)
        dim_customer_latest = dim_customer.where("current = true")

        # get net new rows to insert
        customer_df_insert_net_new = (
            customer_df.join(
                dim_customer_latest,
                (customer_df.id == dim_customer_latest.id)
                & (
                    dim_customer_latest.datetime_updated
                    < customer_df.datetime_updated
                ),
                'leftanti',
            )
            .select(
                customer_df.id,
                customer_df.customer_sur_id,
                customer_df.first_name,
                customer_df.last_name,
                customer_df.state_id,
                customer_df.datetime_created,
                customer_df.datetime_updated,
            )
            .withColumn('current', lit(True))
            .withColumn('valid_from', customer_df.datetime_updated)
            .withColumn('valid_to', lit('2099-01-01 12:00:00.0000'))
        )

        # get rows to insert for existing ids
        customer_df_insert_existing_ids = (
            customer_df.join(
                dim_customer_latest,
                (customer_df.id == dim_customer_latest.id)
                & (
                    dim_customer_latest.datetime_updated
                    < customer_df.datetime_updated
                ),
            )
            .select(
                customer_df.id,
                customer_df.customer_sur_id,
                customer_df.first_name,
                customer_df.last_name,
                customer_df.state_id,
                customer_df.datetime_created,
                customer_df.datetime_updated,
            )
            .withColumn('current', lit(True))
            .withColumn('valid_from', customer_df.datetime_updated)
            .withColumn('valid_to', lit('2099-01-01 12:00:00.0000'))
        )
        # get rows to be updated
        customer_df_ids_update = (
            dim_customer_latest.join(
                customer_df,
                (dim_customer_latest.id == customer_df.id)
                & (
                    dim_customer_latest.datetime_updated
                    < customer_df.datetime_updated
                ),
            )
            .select(
                dim_customer_latest.id,
                dim_customer_latest.customer_sur_id,
                dim_customer_latest.first_name,
                dim_customer_latest.last_name,
                dim_customer_latest.state_id,
                dim_customer_latest.datetime_created,
                customer_df.datetime_updated,
                dim_customer_latest.valid_from,
            )
            .withColumn('current', lit(False))
            .withColumn('valid_to', customer_df.datetime_updated)
        )
        return customer_df_insert_net_new.unionByName(
            customer_df_insert_existing_ids
        ).unionByName(customer_df_ids_update)

    def get_fct_orders(
        self,
        input_datasets: Dict[str, DeltaDataSet],
        spark: SparkSession,
        **kwargs,
    ) -> DataFrame:
        dim_customer = input_datasets['dim_customer'].curr_data
        orders_df = input_datasets['orders'].curr_data

        dim_customer_curr_df = dim_customer.where("current = true")
        return orders_df.join(
            dim_customer_curr_df,
            orders_df.customer_id == dim_customer_curr_df.id,
            "left",
        ).select(
            orders_df.order_id,
            orders_df.customer_id,
            orders_df.item_id,
            orders_df.item_name,
            orders_df.delivered_on,
            orders_df.datetime_order_placed,
            dim_customer_curr_df.customer_sur_id,
        )

    def get_silver_datasets(
        self,
        input_datasets: Dict[str, DeltaDataSet],
        spark: SparkSession,
        **kwargs,
    ) -> Dict[str, DeltaDataSet]:
        self.check_required_inputs(input_datasets, ['customer', 'orders'])
        dim_customer_df = self.get_dim_customer(
            input_datasets['customer'],
            spark,
            dim_customer=spark.read.table(f'{self.DATABASE}.dim_customer'),
        )

        silver_datasets = {}
        silver_datasets['dim_customer'] = DeltaDataSet(
            name='dim_customer',
            curr_data=dim_customer_df,
            primary_keys=['customer_sur_id'],
            storage_path=f'{self.STORAGE_PATH}/dim_customer',
            table_name='dim_customer',
            data_type='delta',
            database=f'{self.DATABASE}',
            partition=kwargs.get('partition', self.DEFAULT_PARTITION),
        )
        self.publish_data(silver_datasets, spark)
        silver_datasets['dim_customer'].curr_data = spark.read.table(
            f'{self.DATABASE}.dim_customer'
        )
        silver_datasets['dim_customer'].skip_publish = True
        input_datasets['dim_customer'] = silver_datasets['dim_customer']

        silver_datasets['fct_orders'] = DeltaDataSet(
            name='fct_orders',
            curr_data=self.get_fct_orders(input_datasets, spark),
            primary_keys=['order_id'],
            storage_path=f'{self.STORAGE_PATH}/fct_orders',
            table_name='fct_orders',
            data_type='delta',
            database=f'{self.DATABASE}',
            partition=kwargs.get('partition', self.DEFAULT_PARTITION),
            replace_partition=True,
        )
        return silver_datasets

    def get_sales_mart(
        self, input_datasets: Dict[str, DeltaDataSet], **kwargs
    ) -> DataFrame:
        dim_customer = (
            input_datasets['dim_customer']
            .curr_data.where("current = true")
            .select("customer_sur_id", "state_id")
        )
        fct_orders = input_datasets['fct_orders'].curr_data
        return (
            fct_orders.alias("fct_orders")
            .join(
                dim_customer.alias("dim_customer"),
                fct_orders.customer_sur_id == dim_customer.customer_sur_id,
                "left",
            )
            .select(
                expr('to_date(fct_orders.delivered_on, "yyyy-dd-mm")').alias(
                    "deliver_date"
                ),
                col("dim_customer.state_id").alias("state_id"),
            )
            .groupBy("deliver_date", "state_id")
            .count()
            .withColumnRenamed("count", "num_orders")
        )

    def get_gold_datasets(
        self,
        input_datasets: Dict[str, DeltaDataSet],
        spark: SparkSession,
        **kwargs,
    ) -> Dict[str, DeltaDataSet]:
        self.check_required_inputs(
            input_datasets, ['dim_customer', 'fct_orders']
        )
        sales_mart_df = self.get_sales_mart(input_datasets)
        return {
            'sales_mart': DeltaDataSet(
                name='sales_mart',
                curr_data=sales_mart_df,
                primary_keys=['deliver_date', 'state_id', 'partition'],
                storage_path=f'{self.STORAGE_PATH}/sales_mart',
                table_name='sales_mart',
                data_type='delta',
                database=f'{self.DATABASE}',
                partition=kwargs.get('partition', self.DEFAULT_PARTITION),
                replace_partition=True,
            )
        }

if __name__ == "__main__":
    spark = (
        SparkSession.builder.appName("adventureworks")
        .enableHiveSupport()
        .getOrCreate()
    )
    spark.sparkContext.setLogLevel("ERROR")
    sm = SalesMartETL()
    partition = datetime.now().strftime(
        "%Y-%m-%d-%H-%M"
    )  # usually from orchestrator -%S
    sm.run(spark, partition=partition)
    spark.stop


# Use this token to access the HTTP API:
# 5620370083:AAGu5OcD-59_sNmPevlZqq8AplOxJkGKwR0

# t.me/Small_test_project_bot

import src.settings as settings # Импортируем "защищенный" файл с "важной" информацией

BOT_TOKEN        = settings.API_KEY # Токен из "защищенного" файла settings.py
DB_PATH          = './TG_bot_for_timesheets/src/bot.db'
CREDENTIALS_FILE = '../src/pythonextension-202bab519501.json'  # Файла "pythonextension-202bab519501.json", содержащий закрытый ключ

from pprint import pprint
import logging
import inspect
import sys, os

# sys.path.insert(0, os.path.abspath('./')) # Добавляем папку выше уровнем для получения их содержимого

# pprint(sys.path)

import dependence.Google_sheet_extension_v3_in_class as GSE
import dependence.sqlite_tg_bot as STB

logging.basicConfig(filename='./TG_bot_for_timesheets/bot.log', level=logging.INFO,
                    format="%(asctime)s %(levelname)s %(message)s" ) # Логгируем ВСЕ сообщения в файл "bot.log" по формату <Врмемя Тип сообщения Сообщение>

def showVarType(var) -> None: # Узнаём тип данных произвольной переменной
    caller_locals = inspect.currentframe().f_back.f_locals
    varName = [name for name, value in caller_locals.items() if var is value] # Получаем имя переменной
    print(var, f'\n{varName} type is {type(var)}') # Узнаём тип данных переменной

# extention = GSE.extension(CREDENTIALS_FILE=CREDENTIALS_FILE,)

DB_handler = STB.tgbot_db(path=DB_PATH)

from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update, ReplyKeyboardMarkup, KeyboardButton

from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, CallbackQueryHandler
import telegram
import telegram.ext

def start(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):

    print("Funcion 'start' was colled !")
    logging.info("Funcion 'start' was colled !")

    keyboard = [ [ InlineKeyboardButton('Let`s start !', callback_data='startCallback')] ] # Создаём кнопку с введённым текстом
    markup = InlineKeyboardMarkup(keyboard) # Создаём разметку с полученной кнопкой
    update.message.reply_text( text='Bot is online !', reply_markup = markup) # Выводим кнопку в чат
    context.user_data['status'] = 'Free' # Записываем сосояние без конкретной привязки к действиям пользователя
    print(f'Context -', context.user_data['status'])

def login(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):

    print("Funcion 'login' was colled !")
    logging.info("Funcion 'login' was colled !")

    print(context.user_data['status'])

    # showVarType(update)
    # userInfo = update.message.chat

    if DB_handler.check_userExistence(update) == False: # Проверяем есть ли такой пользователь в базе данных
        print(f'User with id = {update.callback_query.message.chat.id} not existe')
        DB_handler.set_newUser(update)
        print(f'User with id = {update.callback_query.message.chat.id} now in db')

        set_GoogleSheetID(update, context)
    else: # Такой пользователь уже есть
        print(f'User with id = {update.callback_query.message.chat.id} was existe in db')

        keyboard = [ [ InlineKeyboardButton('Yes', callback_data='set_GoogleSheetID'),  InlineKeyboardButton('No', callback_data='use_extention')] ] # Создаём кнопку с введённым текстом
        markup = InlineKeyboardMarkup(keyboard) # Создаём разметку с полученной кнопкой
        update.callback_query.message.reply_text( text='Do you want to change your Google Sheet ID ?', reply_markup = markup) # Выводим кнопку в чат

        # update.callback_query.message.reply_text(text='Looks like you already was here. \n Please enter your Google sheet ID !', reply_markup = markup )

def set_GoogleSheetID(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Записываем новую ссылку на Google Sheet
    print("Funcion 'set_GoogleSheetID' was colled !")
    logging.info("Funcion 'set_GoogleSheetID' was colled !")

    context.user_data['status'] = 'set_GoogleSheetID' # Записываем сосояние ожидания ввода Google Sheet ID
    old_ID = str(DB_handler.get_userSheetId(update))
    keyboard = [ [ KeyboardButton(old_ID) ] ] # Создаём кнопку с введённым текстом | Нет параметра "Callback_data"
    markup = ReplyKeyboardMarkup(keyboard, resize_keyboard = True, one_time_keyboard = True, input_field_placeholder = "Enter your Google sheet ID here:") # Создаём разметку с полученной кнопкой
    update.callback_query.message.reply_text( text='Please enter your Google sheet ID !', reply_markup = markup) # Выводим кнопку в чат;  Остаток текста - Looks like you already was here. \n
    # showVarType(old_ID)

def insert_Google_Sheet_ID(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):
    print(f"Funcion 'insert_Google_Sheet_ID' was colled with context status {context.user_data['status']} !")
    logging.info(f"Funcion 'insert_Google_Sheet_ID' was colled with context status {context.user_data['status']}")

    DB_handler.set_userSheetId(update) # Записываем новый Google sheet ID в БД для данного пользователя
    check_Google_Sheet_ID(update, context)

def check_Google_Sheet_ID(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext):
    print(f"Funcion 'check_Google_Sheet_ID' was colled with context status {context.user_data['status']} !")
    logging.info(f"Funcion 'check_Google_Sheet_ID' was colled with context status {context.user_data['status']}")

    # status = GSE.extension(CREDENTIALS_FILE = CREDENTIALS_FILE, sheet_id= DB_handler.get_userSheetId(update)).status

    # print(DB_handler.get_userSheetId(update))
    # print(status)

    if GSE.extension(CREDENTIALS_FILE = CREDENTIALS_FILE, sheet_id= DB_handler.get_userSheetId(update)).is_connected() == True:
        keyboard = [ [  ] ] # Создаём кнопку с введённым текстом | Нет параметра "Callback_data"
        markup = ReplyKeyboardMarkup(keyboard, resize_keyboard = True) # Создаём разметку с полученной кнопкой
        update.message.reply_text(text='The bot has successfully connected to your Google sheet !', reply_markup=markup)

        context.user_data['status'] = 'Google' # Записываем сосояние без конкретной привязки к действиям пользователя
        googleHandler(update, context)
    else:
        update.message.reply_text(text='The bot has not successfully connected to your Google sheet !')
        # DB_handler.set_userSheetId(update)

def googleHandler(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Функция обработки запросов к google sheet
    print(f"Funcion 'googleHandler' was colled with context status {context.user_data['status']} !")
    logging.info(f"Funcion 'googleHandler' was colled with context status {context.user_data['status']}")

    pass

def callbackHandler(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Функция обработки обаратных запросов (callback_data)
    global path
    print("Funcion 'callbackHandler' was colled !")
    logging.info("Funcion 'callbackHandler' was colled !")
    query = update.callback_query.data # Вычленяем обаратный запрос (callback_data)
    logging.info(f"Query - '{query}' !")

    # showVarType(context)

    print(f"Query - '{query}' !")

    if query == 'startCallback':
        login(update, context)
    elif query == 'set_GoogleSheetID':
        set_GoogleSheetID(update, context)
    elif query == 'use_extention':
        googleHandler(update, context)

def textHandler(update : telegram.update.Update, context : telegram.ext.callbackcontext.CallbackContext): # Функция обработки текстовых сообщений

    print(f"Funcion 'textHandler' was colled with context status {context.user_data} !")
    logging.info(f"Funcion 'textHandler' was colled with context status {context.user_data} !")

    context_status = context.user_data['status']
    # print(context_status == 'set_GoogleSheetID')
    showVarType(context_status)

    if context_status == 'set_GoogleSheetID':
        print("Go to insert_Google_Sheet_ID !")
        insert_Google_Sheet_ID(update, context)

def photoHandler(update : telegram.update.Update, context): # Функция обработки сообщений с фотографиями
    global path
    print("Funcion 'photoHandler' was colled !")
    logging.info("Funcion 'photoHandler' was colled !")

def set_commandHandlers(mybot : Updater): # Функция обявляет ручки для диспетчра
    dp = mybot.dispatcher # Создаём объект <Диспетчер>

    dp.add_handler(CommandHandler('start', start)) # Ручка для команды "/start"
    dp.add_handler(MessageHandler(Filters.text, textHandler)) # Ручка для всего получаемого текста
    dp.add_handler(MessageHandler(Filters.photo, photoHandler)) # Ручка для всех получаемых фотографий
    dp.add_handler(CallbackQueryHandler(callbackHandler)) # Ручка для обработки различных callback_data
    return dp # Возвращаем диспетчер со всеми "ручками"

def main(strBotToken): # Функция основоного стека вызова
    mybot = Updater(strBotToken, use_context=True) # Создаём объект "mybot" класса "Updater" отвечающий за взаимодействие с сервером telegram

    dp = set_commandHandlers(mybot) # Создаём объект <Диспетчер>

    mybot.start_polling() # Бот начинает обращатся к серверу telegram

    print("-"*10,'\nBot is alive !') # Выводим в консоль что бот запущен
    logging.info('"Bot" is alive !')

    mybot.idle() # бот работает пока работает хост

if __name__ == '__main__': # Если файл tg_bot.py вызаван, то будет запущен main(strBotToken); Если он будет импортироват то ничего не произайдёт
    main(BOT_TOKEN)


# -*- coding: utf-8 -*-
"""
@author: Sanjay Kottapalli (svk4001@med.cornell.edu)
@date: 08/11/2022
Landau Lab, WCM

This script executes functions for the genotyping of cells,
starting from WT and MUT read counts.
"""
#imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
#import sklearn
import timeit

from sklearn.neighbors import KernelDensity
from numpy.polynomial.polynomial import Polynomial
from numpy.polynomial.polynomial import polyfit
from scipy.optimize import minimize
#from scipy.signal import savgol_filter
#from scipy.signal import find_peaks
from scipy.integrate import quad
from collections import Counter
#from sklearn.metrics.pairwise import euclidean_distances
from scipy.stats import gaussian_kde
#from sklearn.metrics import homogeneity_score
#from sklearn.cluster import SpectralClustering
from sklearn.semi_supervised import SelfTrainingClassifier
from sklearn.neighbors import KNeighborsClassifier

#from numpy.random import multivariate_normal
from sklearn.metrics import confusion_matrix
from scipy.stats import gmean

from joblib import Parallel, delayed

import numpy.matlib

import matplotlib as mpl
mpl.rcParams['figure.dpi'] = 500
sns.set(font_scale=1.2)
sns.set_style("ticks")
tab20=plt.get_cmap('tab20')
gen_cmap = {'MUT':tab20.colors[0],'WT':tab20.colors[2],
            'HET':tab20.colors[4],'NA':tab20.colors[6],'-1':tab20.colors[8]}

plt.clf()

def GotchaLabeling(path="", infile="", gene_id="", sample_id=""):
    time1 = timeit.default_timer()
    print("Reading in file.")
    typing = read_data(path+infile, gene_id, sample_id)
    sample_dir = path+sample_id+'/'
    try:
        os.mkdir(sample_dir)
    except:
        pass

    for i in ["WT", "MUT"]:
        print("Noise correcting {} read counts.".format(i))
        if i=="WT":
            typing, wt_min = noise_correct(typing,i,sample_dir,sample_id)
        else:
            typing, mut_min = noise_correct(typing,i,sample_dir,sample_id)

    print("Performing quadrant genotyping.")
    typing = quadrant_genotype(typing, wt_min, mut_min)

    print("Computing KNN-based clusters.")
    typing = KNN_cluster(typing, wt_min, mut_min,
                      0.05, sample_dir)

    typing.to_csv(sample_dir+sample_id+'_genotype_labels.csv')
    print("All analysis complete!")
    time2 = timeit.default_timer()
    print("Total time to execute: {}".format(time2-time1))

    return typing

def read_data(infile="", gene_id="", sample_id=""):
    '''
    This function reads in the sequencing reads from only real cells
    (based on ATAC calling). Cells without GoTChA reads are dropped.
    '''
    cell_line = pd.read_csv(infile, index_col=0, sep=",")
    #print(cell_line.head())
    try:
        genotyping = pd.DataFrame(index=cell_line.index)
        if sample_id in np.unique(cell_line['Sample'].values):
            cell_line = cell_line.loc[cell_line['Sample']==sample_id, :]
        genotyping['WTcount'] = cell_line[gene_id+'_WTcount']
        genotyping['MUTcount'] = cell_line[gene_id+'_MUTcount']
    except:
        cell_line = cell_line.set_index("WhiteListMatch")
        genotyping = pd.DataFrame(index=cell_line.index)
        genotyping['WTcount'] = cell_line['WTcount']
        genotyping['MUTcount'] = cell_line['MUTcount']

    #print(genotyping.head())
    index = genotyping[['WTcount', 'MUTcount']].dropna().index
    genotyping = genotyping.loc[index,:]
    print("Number of cells: "+str(genotyping.shape[0]))

    return genotyping

def noise_correct(typing, feature="", sample_dir="", sample_id=""):
    np.random.seed(0)
    pseudocount = 1
    X = typing[[feature+'count']]+pseudocount
    logged_counts = np.log(X) #change to log10 later, arcsinh? not log2

    #noise = np.random.normal(0,0.0,typing.shape[0]) #no noise
    #logged_counts = logged_counts + noise.reshape(-1,1)
    typing['transf_{}'.format(feature)] = logged_counts

    plt.hist(logged_counts, density=True, bins=50)
    plt.title(feature+' counts')
    plt.ylabel("Probability")
    plt.xlabel("Log(counts+1)")
    #plt.savefig(sample_dir+"log_{}_counts.pdf".format(feature),
    #            dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    bw = 0.1

    kde = KernelDensity(bandwidth=bw)#kernel='exponential')
    kde.fit(typing['transf_{}'.format(feature)].values.reshape(-1, 1))
    x_bin = np.histogram(typing['transf_{}'.format(feature)], bins=50)[1]
    kde_x = np.linspace(min(x_bin)-0.5,max(x_bin)+0.5,10000)
    kde_smooth = np.exp(kde.score_samples(kde_x.reshape(-1, 1)))

    plt.hist(typing['transf_{}'.format(feature)], density=True, bins=50)
    plt.plot(kde_x, kde_smooth, color='red')
    plt.title('Initial KDE')
    plt.ylabel("Probability")
    plt.xlabel("Log(counts+1)")
    plt.savefig(sample_dir+"{}_kde_initial.pdf".format(feature),
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    kde_test = kde_smooth

    def kde_func(x):
        return np.exp(kde.score_samples(x.reshape(1, -1))[0])

    plt.plot(kde_x, np.log(kde_test), color='red')
    plt.title("{} (all drops)".format(sample_id))
    plt.ylabel("Log-Probability")
    plt.xlabel("Log(WTcounts)")
    plt.show()
    plt.clf()

    kde_smooth2 = kde_test#np.log(kde_test)
    #kde_smooth2 = savgol_filter(kde_smooth2, 1001, 3)#, mode='constant', cval=-np.inf)
    plt.plot(kde_x, kde_smooth2, color='red')
    plt.show()
    plt.clf()

    fit = polyfit(kde_x, kde_smooth2, 12)
    poly = Polynomial(fit)
    poly_y = poly(kde_x)
    plt.plot(kde_x, poly_y, color='red')
    plt.show()
    plt.clf()

    mid = (max(kde_x)-min(kde_x))/2
    print(mid)
    result = minimize(poly, x0=np.array([mid]),
                        options={'disp':False}, bounds=((min(kde_x), max(kde_x)),),
                     method='Nelder-Mead')
    new_min = result.x[0]
    print(new_min)

    result = minimize(kde_func, x0=np.array([new_min]),
                        options={'disp':True}, bounds=((min(kde_x), max(kde_x)),),
                     method='Nelder-Mead')
    new_min = result.x[0]
    print(new_min)

    if feature=="WT":
        alt_feature="MUT"
    else:
        alt_feature="WT"

    if new_min==max(kde_x):
        zero_counts = typing['{}count'.format(alt_feature)]
        zero_counts = zero_counts[zero_counts==0].index

        new_min = np.percentile(typing.loc[zero_counts,'transf_{}'.format(feature)], 99.99)

    noise_values = logged_counts
    noise_values = noise_values[noise_values<new_min]
    noise_values = noise_values.dropna()
    prop_noise = len(noise_values)/len(logged_counts)

    kde_noise = KernelDensity(bandwidth=bw)#kernel='exponential')
    kde_noise.fit(noise_values.values.reshape(-1, 1))
    x_bin = np.histogram(typing['transf_{}'.format(feature)], bins=50)[1]
    #kde_x = np.linspace(min(x_bin),max(x_bin),1000)
    noise_smooth = np.exp(kde_noise.score_samples(kde_x.reshape(-1, 1)))

    signal_values = logged_counts
    signal_values = signal_values[signal_values>=new_min]
    signal_values = signal_values.dropna()
    prop_signal = 1-prop_noise

    kde_signal = KernelDensity(bandwidth=bw)#kernel='exponential')
    kde_signal.fit(signal_values.values.reshape(-1, 1))
    #x_bin = np.histogram(typing['transf_{}'.format(feature)], bins=50)[1]
    #kde_x = np.linspace(min(x_bin),max(x_bin),1000)
    signal_smooth = np.exp(kde_signal.score_samples(kde_x.reshape(-1, 1)))

    plt.hist(typing['transf_{}'.format(feature)], density=True, bins=50)
    plt.plot(kde_x, kde_test, color='red')
    plt.plot(kde_x, prop_noise*noise_smooth, color='pink')
    plt.plot(kde_x, prop_signal*signal_smooth, color='yellow')
    plt.title('Noise vs. Signal')
    plt.ylabel("Probability")
    plt.xlabel("Log(counts+1)")
    #plt.xticks(np.arange(0,6,1.0))
    plt.savefig(sample_dir+"{}_kde_mixture.pdf".format(feature),
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    def noise_func(x):
        return np.exp(kde_noise.score_samples(np.array(x).reshape(1, -1)))

    def mean_func(x):
        return x*noise_func(x)

    def var_func(x):
        return x*x*noise_func(x)

    noise_mean = quad(func=mean_func, a=-np.inf, b=np.inf, limit=100)[0]
    noise_var = quad(func=var_func, a=-np.inf,
                     b=np.inf, limit=100)[0] - noise_mean**2
    print("Noise mean, noise var:")
    print(noise_mean, noise_var)

    noise_std = np.sqrt(noise_var)
    #typing['transf_{}'.format(feature)] = (logged_counts-noise_mean)/noise_std

    added_var = 0.3**2
    new_var = noise_var - added_var
    new_std = np.sqrt(new_var)

    typing['transf_{}'.format(feature)] = (np.log(X)-noise_mean)/new_std

    plt.hist(typing['transf_{}'.format(feature)], bins=50, density=True)
    plt.title('{} Z-scores'.format(feature))
    plt.ylabel("Probability")
    plt.xlabel("Z-score")
    plt.savefig(sample_dir+"{}_zscores.pdf".format(feature),
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    feat_min = (new_min-noise_mean)/new_std
    print(feat_min)

    return typing, feat_min

def quadrant_genotype(typing, wt_min, mut_min):
    typing['quadrant_class'] = 'NA'
    for i in typing.index:
        if typing.loc[i,'transf_WT'] < wt_min:
            if typing.loc[i,'transf_MUT'] < mut_min:
                typing.loc[i,'quadrant_class'] = 'NA'
            else:
                typing.loc[i,'quadrant_class'] = 'MUT'
        else:
            if typing.loc[i,'transf_MUT'] < mut_min:
                typing.loc[i,'quadrant_class'] = 'WT'
            else:
                typing.loc[i,'quadrant_class'] = 'HET'

    return typing

def KNN_cluster(typing, wt_min, mut_min,
                      knn_window=0.05, sample_dir=""):

    typing['clusters'] = typing['quadrant_class']

    data = typing[['transf_WT', 'transf_MUT']].values
    range_wt = max(data[:,0])-min(data[:,0])
    range_mut = max(data[:,0])-min(data[:,0])

    indices1 = set(np.where(data[:,0]>wt_min-knn_window*range_wt)[0])
    indices1 = indices1.intersection(set(np.where(data[:,0]<=wt_min+knn_window*range_wt)[0]))
    indices2 = set(np.where(data[:,1]>mut_min-knn_window*range_mut)[0])
    indices2 = indices2.intersection(set(np.where(data[:,1]<=mut_min+knn_window*range_mut)[0]))
    indices = indices1.union(indices2)

    indices = np.array(list(indices))
    indices = typing.index[indices]

    typing.loc[indices,'clusters'] = -1

    n = list(dict(Counter(typing['quadrant_class'].values)).values())
    print(n)
    n_neighbors = round(np.sqrt(gmean(n)))
    print("Nearest neighbors: "+str(n_neighbors))

    print("Remove uncertain labels and re-label with self-training.")

    for label in np.unique(typing['clusters'].astype(str)):
        try:
            plt.scatter(data[np.where(typing['clusters'].astype(str)==label)[0],0],
                        data[np.where(typing['clusters'].astype(str)==label)[0],1], label=label,
                        s=7, color=gen_cmap[label])
        except:
            pass
    handles, labels = plt.gca().get_legend_handles_labels()
    by_label = dict(zip(labels, handles))
    plt.legend(by_label.values(), by_label.keys())
    #plt.legend()
    plt.title('Confident Calls')
    plt.xlabel('WT Z-scores')
    plt.ylabel('MUT Z-scores')
    plt.savefig(sample_dir+"confident_calls.pdf",
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    time1 = timeit.default_timer()
    STC = SelfTrainingClassifier(KNeighborsClassifier(n_neighbors=n_neighbors, weights='distance'),#gauss_func),
                            criterion='k_best', k_best=1, max_iter=None)
    STC.fit(data, typing['clusters'])
    time2 = timeit.default_timer()
    print("Label propagation completed in {} seconds.".format(
        time2-time1))

    cluster_pred = STC.transduction_
    typing['clusters'] = cluster_pred

    typing['genotype_pred'] = typing['clusters']
    del typing['clusters']

    x=data[:,0]#typing.loc[:,['transf_wt']]
    y=data[:,1]#typing.loc[:,['transf_mut']]

    for label in np.unique(typing['genotype_pred'].astype(str)):
        try:
            plt.scatter(data[np.where(typing['genotype_pred'].astype(str)==label)[0],0],
                        data[np.where(typing['genotype_pred'].astype(str)==label)[0],1], label=label,
                       s=7, color=gen_cmap[label])
        except:
            pass
    plt.legend()
    plt.title('Cluster Genotype')
    plt.xlabel('WT Z-scores')
    plt.ylabel('MUT Z-scores')

    plt.axhline(y=mut_min, color='r', linestyle='-')
    plt.axvline(x=wt_min, color='r', linestyle='-')

    #plt.colorbar()
    plt.savefig(sample_dir+"cluster_genotype.pdf",
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    for label in np.unique(typing['quadrant_class']):
        try:
            plt.scatter(data[np.where(typing['quadrant_class']==label)[0],0],
                        data[np.where(typing['quadrant_class']==label)[0],1], label=label,
                       s=7, color=gen_cmap[label])
        except:
            pass
    plt.title('Quadrant Genotype')
    plt.xlabel('WT Z-scores')
    plt.ylabel('MUT Z-scores')

    plt.axhline(y=mut_min, color='r', linestyle='-')
    plt.axvline(x=wt_min, color='r', linestyle='-')

    plt.legend()
    plt.savefig(sample_dir+"quadrant_genotype.pdf",
                dpi=500, bbox_inches = "tight")
    plt.show()
    plt.clf()

    print('Proportion genotyped: (cluster method): ' +
          str(sum(typing['genotype_pred']!='NA')/typing.shape[0]))

    print('Proportion genotyped: (quadrant method): ' +
          str(sum(typing['quadrant_class']!='NA')/typing.shape[0]))

    print("Quadrant labels:")
    print(Counter(typing['quadrant_class']))

    print("Cluster labels:")
    print(Counter(typing['genotype_pred']))

    return typing


from django.shortcuts import render,get_object_or_404, render_to_response, redirect
from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django.views.generic.edit import CreateView
from django.core.exceptions import PermissionDenied
from project.models import Update, Project, Task, Club, Comment, Document
from project.permissions import PermissionHandler
from django.template import RequestContext
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect

# curently using class-based views for cleaner code.
# one view for creating, editing and deleting an object of a particular type.
# the views are then mixed to develop the full context for a given page.
class CreateUpdate( CreateView ):
	model = Update
	fields = ['title','content']

	def form_valid( self, form ):
		if( not PermissionHandler.create_update( self.request.user, **self.kwargs ) ):
			raise PermissionDenied( 'You do NOT have permission to make an update :P' ) # <--- change this to a more tactful message :)

		form.instance.user = self.request.user
		form.instance.parent = Project.objects.filter( pk = self.kwargs['project'] )[0]
		super( CreateUpdate, self ).form_valid( form )
		return HttpResponseRedirect(reverse('project:project_detail',args=[self.kwargs['project']]))

	def get_success_url(self, *args, **kwargs):
		return reverse('project:project_detail',args=[self.kwargs['project']])

class CreateTask( CreateView ):
	model = Task
	fields = ['title','content','deadline','assigned']

	def form_valid( self, form ):
		#if( not PermissionHandler.create_task( self.request.user, **self.kwargs ) ):
		#	raise PermissionDenied( 'You do NOT have permission to make a task :P' )
		form.instance.user = self.request.user
		form.instance.parent = Project.objects.filter( pk = self.kwargs['project'] )[0]
		return super( CreateTask, self ).form_valid( form )

	def get_success_url(self):
		return reverse('project:project_detail',args=[self.kwargs['project']])

class CreateProject( CreateView ):
	model = Project
	fields = ['name','desc','status','budget','year']

	def form_valid( self, form ):
		if( not PermissionHandler.create_project(self.request.user, **self.kwargs ) ): # check for permission using the permissions module.
			raise PermissionDenied( 'You do NOT have permission to make a task :P' )
		form.instance.club = Club.objects.filter( pk = self.kwargs['club'] )[0]
		return super( CreateProject, self ).form_valid( form )

class CreateClub( CreateView ):
	model = Club
	fields = ['name','description']

	def form_valid( self, form ):
		if( not PermissionHandler.create_club(self.request.user, **self.kwargs ) ): # check for permission using the permissions module.
			raise PermissionDenied( 'You do NOT have permission to make a club :P' )
		return super( CreateProject, self ).form_valid( form )

class CreateComment( CreateView ):
	model = Comment
	fields = ['content']

	def form_valid( self, form ):
		if( not PermissionHandler.create_comment(self.request.user, **self.kwargs ) ): # check for permission using the permissions module.
			raise PermissionDenied( 'You do NOT have permission to make a comment :P' )

		form.instance.user = self.request.user
		if( self.kwargs['target_model'] == 'Update'):
			Update.objects.filter( pk = self.kwargs['update'] )[0].comments.add( form.instance )
		elif( self.kwargs['target_model'] == 'Task'):
			Task.objects.filter( pk = self.kwargs['task'] )[0].comments.add( form.instance )

		return super( CreateProject, self ).form_valid( form )

class MyProjectsListView( ListView ):
	template_name = 'project/project_list.html'
	context_object_name = 'project_list'
	model = Project

	def get(self, request,*args,**kwargs):
		if not request.user.is_authenticated():				# check for authentication
			raise PermissionDenied('You\'re not logged in.')

		return super( MyProjectsListView, self ).get( request, *args, **kwargs )

	def get_queryset(self, *args,**kwargs):
		qs = super(MyProjectsListView, self).get_queryset(*args,**kwargs)
		qs.filter( users__in = [self.request.user] )
		return qs

	def get_context_data( self, **kwargs):
		ctx = super( MyProjectsListView, self).get_context_data(**kwargs)
		ctx['page_title'] = 'My Projects'
		return ctx

class AllProjectsListView( ListView ):
	template_name = 'project/project_list.html'
	context_object_name = 'project_list'
	model = Project
	def get_context_data( self, **kwargs):
		ctx = super(AllProjectsListView, self).get_context_data(**kwargs)
		ctx['page_title'] = 'All Projects'
		return ctx

"""
	driver function for the project details view.
"""
@login_required
def project_detail_view( request, *args, **kwargs ):
	# putting together the required context.
	if request.method == 'GET':
	# very basic implementation.
		ctx = { }
		ctx['project'] = get_object_or_404( Project, pk = kwargs['project'] )
		ctx['task_list'] = Task.objects.filter( parent = ctx['project'] )
		ctx['update_list'] = Update.objects.filter( parent = ctx['project'] )
		ctx['documents'] = Document.objects.filter( project = ctx['project'] )
		if Update.objects.filter( parent = ctx['project'] ).count():
			ctx['no_updates'] = 0
		else:
			ctx['no_updates'] = 1

		if Task.objects.filter( parent = ctx['project'] ).count():
			ctx['no_tasks'] = 0
		else:
			ctx['no_tasks'] = 1

		if not PermissionHandler.edit_project( request.user, project = kwargs['project'] ):
			ctx['allow_project_edit'] = 0
		else:
			ctx['allow_project_edit'] = 1

		if not PermissionHandler.create_update( request.user, project = kwargs['project'] ):
			ctx['allow_update'] = 0
		else:
			ctx['allow_update'] = 1

		if not PermissionHandler.create_task( request.user, project = kwargs['project'] ):
			ctx['allow_task'] = 0
		else:
			ctx['allow_task'] = 1

		if not PermissionHandler.add_user_to_project( request.user, project = kwargs['project'] ):
			ctx['allow_add_user'] = 0
		else:
			ctx['allow_add_user'] = 1

		if not PermissionHandler.add_mentor_to_project( request.user, project = kwargs['project'] ):
			ctx['allow_add_mentor'] = 0
		else:
			ctx['allow_add_mentor'] = 1

		return render_to_response( 'project/project_details.html', ctx, context_instance = RequestContext(request) )

	elif request.method == 'POST':
		return HttpResponse('Method inapplicable')
	# user is automatically added by default context processor.

@login_required
def project_update( request, *args, **kwargs ):

	if not PermissionHandler.edit_project( request.user, project = kwargs['project'] ):
		return PermissionDenied('You are not allowed to edit!')

	project = get_object_or_404( Project, pk = kwargs['project'] )

	try:
		project.desc = request.POST['detailed_writeup']
		project.brief = request.POST['brief_writeup']
		project.status = request.POST['status']
		project.name = request.POST['title']
		if('bgImg' in request.FILES):
			project.image = request.FILES['bgImg']
		project.save()
	except ValueError as e:	# implement a form clean mechanism here.
		raise PermissionDenied('Error')

	return redirect( reverse('project:project_detail', args=[kwargs['project']]) )

@login_required
def upload_document( request, *args, **kwargs ):
	if not PermissionHandler.edit_project( request.user, project = kwargs['project'] ):
		return PermissionDenied('You are not allowed to upload to this project!')

	d = Document( doc = request.FILES['doc'], user = request.user, desc = request.POST['desc'] )
	d.save()
	project = get_object_or_404( Project, pk = kwargs['project'] )
	project.documents.add(d)
	project.save()
	return redirect( reverse('project:project_detail', args=[kwargs['project']] ) )

@login_required
def club_detail_view( request, **kwargs ):

	ctx = { }

	ctx['club'] = get_object_or_404( Club, pk = kwargs['club'] )
	ctx['project_list'] = Project.objects.filter( club = ctx['club'] )

	if not PermissionHandler.edit_club( request.user, club = kwargs['club'] ):
		ctx['allow_club_edit'] = 0
	else:
		ctx['allow_club_edit'] = 1

	if not PermissionHandler.create_project( request.user, club = kwargs['club'] ):
		ctx['allow_project_create'] = 0
	else:
		ctx['allow_project_create'] = 1

	if not PermissionHandler.add_core_to_club( request.user, club = kwargs['club'] ):
		ctx['allow_add_core'] = 0
	else:
		ctx['allow_add_core'] = 1

	return render_to_response( 'project/club_details.html', ctx, context_instance = RequestContext(request) )

@login_required
def club_update( request, *args, **kwargs ):

	if not PermissionHandler.edit_club( request.user, club = kwargs['club'] ):
		return PermissionDenied('You are not allowed to edit!')

	club = get_object_or_404( Club, pk = kwargs['club'] )

	try:
		club.description = request.POST['description']
		club.name = request.POST['name']
		club.image = request.FILES['image']
		club.save()
	except ValueError as e:	# implement a form clean mechanism here.
		raise PermissionDenied('Error')

	return redirect( reverse('project:club_detail', args=[kwargs['club']] ) )

@login_required
def create_default_project( request, *args, **kwargs ):
	pass

@login_required
def create_default_club( request, *args, **kwargs ):
	pass


import glob
import json
import os
import shutil
from datetime import datetime

import imagesize
from detectron2.config import get_cfg
from detectron2.data.datasets import register_coco_instances
from detectron2.engine import DefaultPredictor
import numpy as np
import torch
import matplotlib.gridspec as gridspec

def simple_filename(filename_ext):
    filename_base = os.path.basename(filename_ext)
    filename_noext = os.path.splitext(filename_base)[0]
    return filename_noext

def model_loader(settings):
    cfg = get_cfg()
    cfg.merge_from_file(settings["yaml_path"])
    cfg.MODEL.WEIGHTS = settings["pkl_path"]
    # cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5
    model = DefaultPredictor(cfg).model
    model.eval()
    for param in model.parameters():
        param.requires_grad = False
    return model, cfg

def pick_coco_exp(name, targetlist):
    if os.path.isdir(name):
        shutil.rmtree(name)
    os.makedirs(name, exist_ok=True)

    coco_path = os.environ["DETECTRON2_DATASETS"]
    anno_path = "./dataset/annotations/instances_OpenImage_v6.json"

    file_list = glob.glob(os.path.join(coco_path, "*.jpg"))
    file_list = [x for x in file_list if simple_filename(x) in targetlist]

    file_name_list = [os.path.basename(x) for x in file_list]
    with open(anno_path, "r") as anno_file:
        coco_json = json.load(anno_file)
    my_json = {}
    my_json["info"] = coco_json["info"]
    my_json["licenses"] = coco_json["licenses"]
    my_json["images"] = []
    my_json["annotations"] = []
    my_json["categories"] = coco_json["categories"]

    my_json["images"].extend(
        [x for x in coco_json["images"] if x["file_name"] in file_name_list]
    )
    image_id_list = [x["id"] for x in my_json["images"]]
    my_json["annotations"].extend(
        [x for x in coco_json["annotations"] if x["image_id"] in image_id_list]
    )

    for filepath in file_list:
        shutil.copy(filepath, name)
    with open(f"{name}/my_anno.json", "w") as my_file:
        my_file.write(json.dumps(my_json))
    register_coco_instances(name, {}, f"{name}/my_anno.json", name)

def print_settings(settings, index):
    model_name = settings["model_name"]
    VTM_param = settings["VTM"]
    print()
    print("Evaluation of proposed methods for:", model_name.upper())
    print(f"Settings ID: {index}")
    print(f"VTM paramerters      : {VTM_param}")

import cv2
import numpy as np

def save_feature_map(filename, features):
    features_draw = features.copy()
    del features_draw["p6"]
    _save_feature_map(filename, features_draw)

def _save_feature_map(filename, features, debug=False):
    feat = [features["p2"].squeeze(), features["p3"].squeeze(), features["p4"].squeeze(), features["p5"].squeeze()]
    width_list = [16, 32, 64, 128]
    height_list = [16, 8, 4, 2]
    tile_big = np.empty((0, feat[0].shape[2] * width_list[0]))
    for blk, width, height in zip(feat, width_list, height_list):
        big_blk = np.empty((0, blk.shape[2] * width))
        for row in range(height):
            big_blk_col = np.empty((blk.shape[1], 0))
            for col in range(width):
                tile = blk[col + row * width].cpu().numpy()
                if debug:
                    cv2.putText(
                        tile,
                        f"{col + row * width}",
                        (32, 32),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        0.5,
                        (255, 255, 255),
                        1,
                    )
                big_blk_col = np.hstack((big_blk_col, tile))
            big_blk = np.vstack((big_blk, big_blk_col))
        tile_big = np.vstack((tile_big, big_blk))
    tile_big = tile_big.astype(np.uint16)
    cv2.imwrite(filename, tile_big)

def result_in_list(settings, number, result, set_index):
    res = list(result.values())[0]
    ap = res["AP"]
    ap50 = res["AP50"]
    aps = res["APs"]
    apm = res["APm"]
    apl = res["APl"]

    return [
        datetime.now(),
        set_index,
        number,
        f"{ap:.3f}",
        f"{ap50:.3f}",
        f"{aps:.3f}",
        f"{apm:.3f}",
        f"{apl:.3f}",
    ]

def get_size(start_path = '.'):
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(start_path):
        for f in filenames:
            fp = os.path.join(dirpath, f)
            # skip if it is symbolic link
            if not os.path.islink(fp):
                total_size += os.path.getsize(fp)

    return total_size


from django.forms import ModelForm, DateInput
from walkie_talkie.models import WalkieTalkie

class WalkieTalkieForm(ModelForm):

    class Meta:
        model = WalkieTalkie
        fields = '__all__'
        labels = {'id_organization': 'Название организации'}

        widgets = {
            'date_of_purchase': DateInput(
                format='%d-%m-%Y',
                attrs={'type': 'datetime-local'
                       }),
            'date_of_received': DateInput(
                format='%d-%m-%Y',
                attrs={'type': 'datetime-local'
                       }),
        }


import json
import requests
from urllib.request import urlopen
import pandas as pd
import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import dash_bootstrap_components as dbc
import dash
from dash.dependencies import Input, Output
import dash_auth

USERNAME_PASSWORD_PAIRS = [['username','password'],['kopisusu','1998']]

def pprint(string):
  print(json.dumps(string, indent=2))

result = requests.get("http://192.168.18.110:14240/restpp/query/MyGraph/hobby_param")

hasil = json.loads(result.text)
pprint(hasil)
interest =[]
avg_age = []

for item in hasil["results"][0]["tmp"]:
    interest.append(item['attributes']['interest'])
    avg_age.append(item['attributes']['@avg'])

print (interest,avg_age)

dff = pd.DataFrame({
    "rata2umur": avg_age,
    "hobby": interest,

})

#params= {'id':"\"\"error\"\""}

# req =  requests.get("http://192.168.1.90:14240/restpp/query/mygraph/ip_tracking",params=params)

########### INI UNTUK MENGAMBIL DATA DARI RESPP API TIGERGRAPH ###############

with urlopen("http://192.168.18.110:14240/restpp/query/MyGraph/most_followers") as response:
    data = response.read()

########### LALU DATA TERSEBUT KITA MASUKAN KEDALAM VARIABLE PYTHON ##################
source = json.loads(data)
f_name=[]
num_follower =[]
print (source)

############# DIMANA KITA HARUS MENGGUNAKAN LOGIC LOOPING UNTUK MEMBUATNYA ############

for a in source['results'][0]['followers']:
    f_name.append(a['attributes']['first_name'])
    num_follower.append (a['attributes']['@num_followers'])
    # print(f_name,num_follower)

############## SETELAH ITU DATA TERSEBUT DI MASUKAN KEDALAM DATAFRAME PANDAS #############
df1 = pd.DataFrame({
    "first_name": f_name,
    "count_follow": num_follower,

})

######### INI UNTUK MENGECEK APAKAH DATA TERSEBUT UDAH SIAP UNTUK KITA INPUT KEDALAM PARAMETER DASH #############
# print (df1)

print ('===========================================')

def pprint(string):
  print(json.dumps(string, indent=2))

############# INI QUERY KE DUA YANG MEMAKAI PARAMETER DIDALAMNYA #############

params = {
  "v_type": "users",
  "e_type": "reffered_by",
  "max_change": 0.001,
  "max_iter": 25,
  "damping": 0.85,
  "top_k": 100,
  "print_accum": True,
  "result_attr": "",
  "file_path": "",
  "display_edges": False
}

req = requests.get("http://192.168.18.110:14240/restpp/query/MyGraph/pagerank",params=params)

data2 = json.loads(req.text)

q1= data2['results'][0]['@@topScores']
vertex_id= []
score = []

# print (q1)
for i in q1:
    vertex_id.append(i['Vertex_ID'])
    score.append(i['score'])
    # print (vertex_id,score)

df = pd.DataFrame({
    "v_id": vertex_id,
    "val_score": score,
})

print(df)
print(df1)
# df=pd.DataFrame(q1)

# df = pd.DataFrame(req)
# print (df)

############ INI KITA MEMASUKAN DATA YANG SUDAH KITA BUAT KEDALAM DATAFRAME UNTUK SIAP DI VISUALISASIKAN ######

fig = px.bar(df1, x="first_name", y="count_follow", color="first_name")
fig1 = px.bar( df, x = "v_id", y= "val_score", )
fig3= px.pie (dff,values= "rata2umur",color_discrete_map= "hobby",names="hobby",)

###### MULAI DARI SINI KITA MEMBUAT LAYOUT UNTUK MEMBUAT SEBUAH VISUALISASI ##########

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.SUPERHERO],suppress_callback_exceptions=True)
# app = dash.Dash(__name__)
auth = dash_auth.BasicAuth(app,USERNAME_PASSWORD_PAIRS)
#

colors = {
    'background': '#111111',
    'text': '#7FDBFF'
}

fig3.update_layout (
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)

fig.update_layout (
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)

fig1.update_layout(
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)

PLOTLY_LOGO = "https://i.ibb.co/9Vp8QKx/Untitled-design-9-1.png"
titlecard = dbc.Card([
  dbc.CardBody([
    dbc.Row([dbc.Col(html.Center(html.H1("VISUALISASI VIRTUAL-BOX")),style={'fontColor':'text'}),
    dbc.Col(html.Img(src=PLOTLY_LOGO, width="225px" ,style={"right":0,"margin-bottom": "14px",}))]),
  ])
], color="#ff9100",
   style={
     "width":"70rem",
     "margin-top":"1rem",
     "margin-bottom":"1rem",
     "margin-right":"0rem"

   })

piechartcard = dbc.Card([
  dbc.CardBody([
    html.H1("Rata-Rata Umur hobby", className='card-title'),
    html.P("", className='card-body'),
    html.Label(['Choose column:'],style={'font-weight': 'bold'}),
    dbc.DropdownMenuItem(dcc.Dropdown(id="dropdown",multi= True,
       options=[{'label':x,'value': x}for x in sorted(dff.hobby)],
       value= ["running","karaoke"],),style={"width":"65rem","color":"#014c63"}),
    html.Br(),
    dcc.Graph(id='graph', figure= {},),
  ])
],
  outline=True,
  color='secondary',
  style={
    "width":"71rem",
    "margin-right":"0rem",
    "margin-left":"1rem",
    "margin-bottom":"1rem"
  }
)

markdown = ''' Most Follower
            Grafik Diatas merupakan hasil Dari algoritma Query
            untuk menghitung berapa user account yang mempunyai follower terbanyak
            Grafik ini merupakan hasil dari Graph Schema social media
            yang dimana merepresentasikan keterhubungan antar user account
  '''
# piechartcard = dbc.Card([
#   dbc.CardBody([
#     html.H1("Rata-Rata Umur hobby", className='card-title'),
#     html.P("", className='card-body'),
#     dcc.Graph(id='graph', figure= fig3)
#   ])
# ],
#   outline=True,
#   color='secondary',
#   style={
#     "width":"70rem",
#     "margin-right":"0rem",
#     "margin-left":"2rem",
#     "margin-bottom":"1rem"
#   }
# )

bar = html.Div([html.Header("Most Followers"),
                dcc.Graph(id='bar', figure= fig)
              ])

bar1 = html.Div([html.Header("PageRank"),
                 dcc.Graph(id='barchart' ,figure=fig1)
])

barchartcard = dbc.Card([
  dbc.CardBody([
    html.H1("Most Followers", className='card-title'),
    html.P("", className='card-body'),
    dcc.Graph(id='bar', figure= fig)
  ])
],
  outline=True,
  color='secondary',
  style={
    "width":"35rem",
    "margin-right":"0rem",
    "margin-left":"1rem",
    "margin-bottom":"1rem"
  }
)

barchartcard1 = dbc.Card([
  dbc.CardBody([
    html.H1("Algoritma Pagerank", className='card-title'),
    html.P("", className='card-body'),
    dcc.Graph(id='barchart', figure= fig1)
  ])
],
  outline=True,
  color='secondary',
  style={
    "width":"35rem",
    "margin-right":"0rem",
    "margin-left":"1rem",
    "margin-bottom":"1rem"
  }
)
SIDEBAR_STYLE = {
    "position": "fixed",
    "top": 0,
    "left": 0,
    "bottom": 0,
    "width": "16rem",
    "padding": "2rem 1rem",
    "background-color": "#424242",
}

WHITE = "#fffff8"
TG_LOGO = "https://i.pinimg.com/originals/98/c7/e9/98c7e95ea1207eef6f70f0ae9e694e67.png"
sidebar =dbc.Card([html.Div(
    [
        html.H2("Main Menu", className="display-5"),
        html.Hr(),
        html.Label(
            "TigerGraph as a GraphDatabase", className="display-6"
        ),
        html.Hr(),
        dbc.Nav(
            [
                dbc.NavLink("Home", href="/", active="exact"),
                dbc.NavLink("Page 1", href="/page-1", active="exact"),
                dbc.NavLink("Page 2", href="/page2", active="exact"),
                dbc.NavLink("Page 3", href="/page3", active="exact"),
            ],
            vertical=True,
            pills=True,

        ),
        html.Hr(),
        html.Li(html.A("Inroduction to Graphs", href='https://www.tigergraph.com/blog/what-is-a-graph-database-and-why-should-you-care/', target="_blank", style={'color':WHITE}), style={'color':WHITE}),
        html.Br(),
        html.Li(html.A("Introduction to Dash", href='https://www.youtube.com/watch?v=e4ti2fCpXMI', target="_blank", style={'color':WHITE}), style={'color':WHITE}),
        html.Br(),
        html.Br(),
        html.Br(),
        html.Br(),
        html.Br(),
        html.Label(["Achmad Irfandi Darmawan"],style= {"fontcolor":"black"})
    ],
    style=SIDEBAR_STYLE,
    className="card"
)])

page = html.Center(html.Div([
  titlecard,
  dbc.Row([barchartcard,
    barchartcard1,piechartcard
  ],justify='center'),

]
, style={"width":"90rem"}))

# page1 = html.Center(html.Div([
#   titlecard,
#   dbc.Row(dbc.Col([
#     bar
#   ],width={'size': 12, "offset": 0}),)

# ]
# , ))

page1 =dbc.Card([
        dbc.CardBody([
        html.H1("Most Follower", className='card-title'),
        html.P("", className='card-body'),
        bar,
        html.Div([html.Label(dcc.Markdown(children=markdown),className="display-5")])
          ])
        ],outline=True,
  color='info',
  style={
    "width":"89rem",
    "margin-right":"1rem",
    "margin-left":"18rem",
    "margin-bottom":"1rem",
    "margin-top":"5rem"
  }),

page2 =dbc.Card([
        dbc.CardBody([
        html.H1("PageRank", className='card-title'),
        html.P("", className='card-body'),
        bar1,
        html.Div()
          ])
        ],outline=True,
  color='info',
  style={
    "width":"89rem",
    "margin-right":"1rem",
    "margin-left":"18rem",
    "margin-bottom":"1rem",
    "margin-top":"5rem"
  }),
dcc.Markdown('''# Most Follower

               ## Grafik Diatas merupakan hasil Dari algoritma Queri
               untuk menghitung berapa user account yang mempunyai
               follower terbanyak

               ## Grafik ini merupakan hasil dari Graph Schema social media
               yang dimana merepresentasikan keterhubungan antar user account
  ''')

# page2= html.Center(html.Div([
#   titlecard,
#   dbc.Row([
#     barchartcard1,
#   ],justify='center'),

# ]
# , style={"width":"90rem"}))

page3= html.Center(html.Div([

  dbc.Row([
    piechartcard,
  ],justify='center'),

]
, style={"width":"90rem"}))

CONTENT_STYLE = {
    "margin-left": "2rem",
    "margin-right": "0rem",
    "padding": "1rem 1rem",
}

# app.layout= html.Div([
#   sidebar,
#   html.Div(page),

# ])

app.layout = html.Div([
    dcc.Location(id='url', refresh=False),
    sidebar,
    html.Div(id='page-content', style=CONTENT_STYLE)
])

@app.callback(dash.dependencies.Output('page-content', 'children'),
              [dash.dependencies.Input('url', 'pathname')],suppress_callback_exceptions=True)

def display_page(pathname):
  if pathname == "/":
    return page
  elif pathname == "/page-1":
    return page1
  elif pathname == "/page2":
    return page2
  elif pathname == "/page3":
    return page3

@app.callback(Output(component_id="graph", component_property='figure'),
[Input(component_id='dropdown', component_property='value')],suppress_callback_exceptions=True)

def update_my_graph(val_chosen):
  if len(val_chosen)>0:
    print(f"value user choose:{val_chosen}")
    print (type(val_chosen))
    dff1 = dff[dff["hobby"].isin(val_chosen)]
    pig = px.pie(dff1,values="rata2umur", names="hobby")
    pig.update_layout(
    plot_bgcolor=colors['background'],
    paper_bgcolor=colors['background'],
    font_color=colors['text']
)
    return pig

# style= CONTENT_STYLE

if __name__ == '__main__':
    app.run_server(debug=True)


# lstm
from keras.models import Sequential
from keras.layers.convolutional import Conv3D
from keras.layers.convolutional_recurrent import ConvLSTM2D
from keras.layers.normalization import BatchNormalization
from numpy import concatenate
import numpy as np
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from pandas import DataFrame, Series
from sklearn.preprocessing import MinMaxScaler
from keras import backend as K
data = np.loadtxt('D:/干旱/data/result8/qm_predict.txt') #month *[pts*lead*model]
row = 65
col = 37
pts = row*col
month = data.shape[0]
lead = 9
model_num = 8

index_nan = np.where(np.isnan(data))
data[index_nan] = 0

# normalize features
scaler = MinMaxScaler(feature_range=(0, 1))
data_scaled = scaler.fit_transform(data)

data = np.reshape(data_scaled, (month, pts, lead, model_num), order='F')

start_year = 1982
end_year = 2016
train_end_year = 2010
train_end = (train_end_year - start_year + 1)*12
# all = (end_year - start_year + 1)*12
year = (end_year - start_year + 1)

gpcc_china = np.loadtxt('D:/干旱/data/result8/gpcc_china.txt') #month*pts
index_nan = np.where(np.isnan(gpcc_china))
gpcc_china[index_nan] = 0
gpcc_scaled = scaler.fit_transform(gpcc_china)

lstm_predict = np.zeros((month, pts, lead), dtype=np.float)
for i in range(pts):
    for j in range(lead):
        obs = gpcc_scaled[:, i]
        model = data[:, i, j, :]
        model = np.reshape(model, (month, model_num), order='F')
        train_x = model[0:train_end, :]
        train_y = obs[0:train_end]
        test_x = model

        if sum(train_y) == 0:
            continue

        # reshape input to be 3D [samples, timesteps, features]
        train_x = train_x.reshape(( train_end, 1, model_num), order='F')
        train_y = train_y.reshape(( train_end, 1), order='F')
        test_x = test_x.reshape(( month, 1, model_num), order='F')

        model = Sequential()
        model.add(LSTM(10, input_shape=(train_x.shape[1], train_x.shape[2])))  # 50
        model.add(Dense(1))
        model.compile(loss='mse', optimizer='adam')
        # fit network
        history = model.fit(train_x, train_y, epochs=1000, \
                            batch_size = train_x.shape[0], verbose=2, shuffle=False, validation_split=0.1) #train_x.shape[0]
        y_pre = model.predict(test_x)
        lstm_predict[:, i, j] = y_pre[:, 0]
        print ("%d %d"%(i,j))
        # destroy the current TF graph and create a new one
        K.clear_session()
        cfg = K.tf.ConfigProto()
        cfg.gpu_options.allow_growth = True
        K.set_session(K.tf.Session(config=cfg))

# inverse transform
for i in range(lead):
    lstm_predict[:, :, i] = scaler.inverse_transform(lstm_predict[:, :, i])

lstm_predict = lstm_predict.reshape((month,pts*lead), order='F')
np.savetxt('D:/干旱/data/result8/lstm_predict.txt', lstm_predict)


# Gambiarras
import sys
this_dir = sys.path[0]
import matplotlib
matplotlib.use('tkagg')

#Importação das bibliotecas de interesse:
import numpy as np
import math
import matplotlib.pyplot as plt
from scipy import signal
import os

def funcao_gera_figura(f,s,t,nome):
    #ATENÇÃO! NÃO HÁ NECESSIDADE DE ALTERAR ESTA FUNÇÃO.

    #Entradas:
    #f [numpy array] : função periódica que estamos interessados
    #em analisar
    #s [numpy array]: expansão em Série de Fourier da função f, pre-
    #viamente calculada
    #t [numpy array]: vetor de tempo
    #nome [string]: nome do arquivo a ser salvo.
    #Importação das bibliotecas de interesse:

    #Resolução da figura:
    dpi = 300
    #Criação da figura, com tamanho e resolução especificados
    fig, ax = plt.subplots()
    fig.set_size_inches(5, 3)
    fig.set_dpi(dpi)

    #Gera os gráficos da função de interesse e da sua aproximação
    #por Série de Fourier:
    plt.plot(t,f,'-b')
    plt.plot(t,s,'-r')
    #Definição dos eixos
    ax.axis([min(t), max(t), -0.5, 1.5])
    #Gera a grade para facilitar a leitura do gráfico
    ax.grid()
    #Nomes de cada eixo:
    plt.xlabel(r'$t[s]$')
    #Salva a imagem em formato pdf.
    plt.savefig(f'{nome}.jpg', format='jpg', dpi=dpi)

def coeficientes_fourier(numero_de_harmonicas):
    #Entrada:
    #numero_de_harmonicas [int]: numero de harmônicas a serem consi-
    #deradas
    #Saída:
    #a_n [numpy array]: vetor de coeficientes a_n (n entre 1 e
    #numero_de_harmonicas) da expansão em Série de Fourier

    an = np.zeros(numero_de_harmonicas+1)
    an[0] = 0.5

    for n in range(1, numero_de_harmonicas+1):
        an[n] = 0.0 if n % 2 == 0 else (2/(n*np.pi)) * (-1)**((n-1)/2)
    return an

# Script principal:
t = np.linspace(-2, 2, 10000)

T0 = 2

f0 = 1/T0

f = 0.5*(signal.square(2*np.pi*f0*(t+T0/4))+np.ones_like(t))

a0 = 0.5

vetor_harmonicas = [1,3,5,7,9,11,13,25,50]

for N in vetor_harmonicas:
    an = coeficientes_fourier(N)
    s = a0 * np.ones_like(t)
    for n in range(1,N+1):
        s += an[n] * np.cos(n*2*np.pi*f0*t)

    nome_figura = os.path.join(this_dir, f"quadrada_{N}_harmonicas")
    funcao_gera_figura(f,s,t,nome_figura)


from typing import List

class Solution:
    def minSubArrayLen(self, s: int, nums: List[int]) -> int:
        nums_sum = 0
        min_sub_len = 0
        left_index = 0
        sum_dict = {}

        for index, element in enumerate(nums):
            nums_sum += element
            sum_dict[index] = nums_sum
            if not min_sub_len and nums_sum >= s:
                min_sub_len = index + 1

        if not min_sub_len:
            return 0

        for index, element in enumerate(nums):
            if sum_dict[index] - sum_dict[left_index] >= s:
                min_sub_len = min(min_sub_len, index - left_index)
                while sum_dict[index] - sum_dict[left_index + 1] >= s:
                    left_index += 1
                    min_sub_len = min(min_sub_len, index - left_index)

        return min_sub_len

class Solution2:
    def minSubArrayLen(self, s: int, nums: List[int]) -> int:
        sum_nums = 0
        left_index = 0
        sub_len = len(nums) + 1

        for index, element in enumerate(nums):
            sum_nums += element
            while sum_nums >= s:
                sub_len = min(sub_len, index + 1 - left_index)
                sum_nums -= nums[left_index]
                left_index += 1
        return sub_len if sub_len <= len(nums) else 0


"""
Renderer object
"""

import pygame
import scripts.utilities as utilities

class Renderer:
    def __init__(self, app) -> None:

        """Initialize Renderer object.

        Keyword arguments:
        app - Main application class
        """

        self.app = app
        self.config = app.config

    def render(self) -> None:

        """Renders everything"""

        self.surface = self.app.window.app_window
        size = self.app.vrt_space.size
        for i in range(30):
            for j in range(30):
                if (i + j) % 2 == 0:
                    pygame.draw.rect(self.surface, (30, 30, 30), (i * size, j * size, size, size))

        self.app.vrt_space.food.draw(self.surface)
        for worm in self.app.vrt_space.worms[:self.app.game_instance.active_players]:
            worm.draw(self.surface)

        # Show debug values
        utilities.debug(self.surface, self.app.window.font, (750, 10), self.app.window.fps)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from mock import create_autospec, MagicMock, patch
from hypothesis.extra.django import TestCase
from hypothesis.extra.django.models import models as djangoModels
from django.contrib.auth.models import User
from django.db import transaction
from django.utils import timezone
from urllib import quote_plus
from hypothesis import given, assume, note, settings as hypothesisSettings
import math
from hypothesis.strategies import characters, just, sampled_from, emails, floats, composite, lists, dictionaries, one_of, text, none, fixed_dictionaries
import string
from geopy.location import Location
from geopy.point import Point
from . import models, exporting
from rest_framework.test import APITestCase
import pytest
import functools
from django.urls.base import reverse

geocodeDict = {
    'raw': text(),
    'street_number': text(),
    'route': text(),
    'locality': text(),
    'postal_code': text(alphabet=('-',) + tuple(string.digits)),
    'state': text(),
    'country': text(),
    'neighborhood': text(),
    'lat': floats(),
    'lng': floats()
}

geocodeTest = \
    one_of(
        dictionaries(text(), one_of(text(), none())),
        fixed_dictionaries(geocodeDict)
    )

@pytest.fixture
def person():
    return models.Person.objects.get_or_create(name='',
            email='test@example.com')[0]

@composite
def peopleArgs(draw):
    return draw(one_of(
        fixed_dictionaries(dict(
            name=one_of(just(None), text()),
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
            address=one_of(just(None), text()),
        )),
        fixed_dictionaries(dict(
            name=one_of(just(None), text()),
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
        )),
        fixed_dictionaries(dict(
            name=one_of(just(None), text()),
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
        )),
        fixed_dictionaries(dict(
            email=emails().filter(lambda x: len(x) < 100 and '/' not in x),
        ))))

@composite
def people(draw):
    return models.Person.objects.create(
        name=draw(nonblanks()),
        email=draw(emails().filter(lambda x: len(x) < 100 and '/' not in x)),
        address=None,
    )

@composite
def nonblanks(draw):
    return draw(text(min_size=1, alphabet=characters(whitelist_categories=('Lu', 'Ll'))))

@composite
def latlngs(draw):
    return {
        'lat': draw(floats(min_value=-90, max_value=90)),
        'lng': draw(floats(min_value=-180, max_value=180))
    }

@composite
def googleAddressTypes(draw):
    return draw(lists(sampled_from((
        'locality', 'street_number', 'route', 'postal_code', 'state',
        'administrative_area_level_1',
        'administrative_area_level_2',
        'administrative_area_level_3',
        'administrative_area_level_4',
        'administrative_area_level_5',
        'neighborhood', 'country', 'intersection', 'political',
        'colloquial_area', 'ward', 'sublocality', 'premise', 'subpremise',
        'natural_feature', 'airport', 'park', 'point_of_interest', 'floor',
        'establishment', 'parking', 'post_box', 'postal_town', 'room',
        'bus_station', 'train_station', 'transit_station'
    )), min_size=1, unique=True))

@composite
def googleAddressComponents(draw):
    return {
        'types': draw(googleAddressTypes()),
        'long_name': draw(text()),
        'short_name': draw(text())
    }

@composite
def googleResponses(draw):
    return {
        'formatted_address': draw(text()),
        'place_id': draw(text()),
        'geometry': {
            'location': draw(latlngs()),
            'viewport': {
                'northeast': draw(latlngs()),
                'southwest': draw(latlngs())
            },
            'bounds': {
                'northeast': draw(latlngs()),
                'southwest': draw(latlngs())
            }
        },
        'types': draw(googleAddressTypes()),
        'address_components': draw(lists(googleAddressComponents(), min_size=1))
    }

@composite
def locations(draw):
    resp = draw(googleResponses())
    asPoint = Point(resp['geometry']['location']['lat'], resp['geometry']['location']['lng'])
    return Location(resp['formatted_address'], asPoint, resp)

def assertValidResponse(resp, status=200):
    __tracebackhide__ = True
    assert resp.status_code == status
    return resp

@pytest.mark.django_db
def testPermissions(api_client, person):
    """Ensure that we do not have access to PII by default"""
    assertValidResponse(api_client.get('/api/people/'), 403)
    assertValidResponse(api_client.get('/api/people/'+quote_plus(person.email)+'/'), 403)

@pytest.mark.skip_auth
@pytest.mark.django_db
@given(emails().filter(lambda e: '/' not in e))
@hypothesisSettings(deadline=None)
def testValidEmailURLs(api_client, email):
    """Test that all permutations of an email address generate a proper url"""
    url = reverse('person-detail', kwargs={'email': email})
    assertValidResponse(api_client.get(url), 404)

@pytest.mark.skip(reason="Buggy under pytest")
@pytest.mark.skip_auth
@pytest.mark.django_db
@given(person=peopleArgs(), newTag=nonblanks())
def testCreatePersonWithTag(api_client, settings, person, newTag):
    note('Email %s'%person['email'])
    person['tags'] = [newTag]
    note("Submit %r"%(person,))
    resp = assertValidResponse(api_client.post('/api/people/',
            person, format='json'), 201).data
    note('Result %r'%(resp,))
    personName = person.get('name', None)
    if personName is not None:
        personName = personName.strip()
    else:
        personName = ''
    assert resp['tags'] == person['tags']
    assert resp['email'] == person['email']
    assert resp['name'] == personName

    person['tags'].append(newTag+'-added')
    resp = assertValidResponse(api_client.put('/api/people/'+person['email']+'/',
            person, format='json'), 200).data
    assert resp['tags'] == person['tags']

@pytest.mark.django_db
def testExportPersonToMailchimp(person, settings):
    settings.MAILCHIMP_SECRET_KEY = '0' * 32
    settings.MAILCHIMP_LIST_ID = 'list-id'
    with patch('mailchimp3.entities.listmembers.ListMembers.create_or_update') as patched:
        exporter = exporting.MailchimpExporter()
        assert len(exporter) == 1
        page = iter(exporter).next()
        assert page.dict[0]['email'] == person.email
        exporter.export_page(page)
        patched.assert_called_with('list-id', '55502f40dc8b7c769880b10874abc9d0',
                dict(email_address=person.email, status_if_new='subscribed'))


from os import fsdecode
from xml.dom.minidom import Document
from searchindex.buildindex import index
from searchindex.preprocessing import preprocess_text
import re
from collections import defaultdict
from math import log10
from searchengine.models import Document
def run_ranked_search(query):
    results = defaultdict(float)

    query_preproc = preprocess_text(query)

    for term in query_preproc:
        if term in index:
            for doc_id in index[term]:

                tf = len(index[term][doc_id])
                df = index[term].doc_freq
                if (tf != 0) and (df != 0):
                    w = (1 + log10(tf)) * log10(index.doc_count / df)

                results[doc_id] += w
    return sorted(results, key=results.get, reverse=True)[:50]

def listToString(s):

    # initialize an empty string
    str1 = ""

    # traverse in the string
    for ele in s:
        str1 += ele

    # return string
    return str1

def run_query(query):

    match = "#(\d+)\((\w+), (\w+)\)"
    frstprt = []
    scndprt = []
    spl = query.split()

    if ' AND NOT ' in query:
        frstprt, scndprt = query.split('AND NOT')
        frstprt = preprocess_text(frstprt)[0]
        scndprt = preprocess_text(scndprt)[0]
        frstprt = index[frstprt]
        scndprt = index[scndprt]
        result = [i for i in frstprt if i not in scndprt]
        return result

    elif ' AND ' in query:
        frstprt, scndprt = query.split('AND')
        frstprt = preprocess_text(frstprt)[0]
        scndprt = preprocess_text(scndprt)[0]
        frstprt = index[frstprt]
        scndprt = index[scndprt]
        print("LOG, AND", frstprt, scndprt)
        result = [i for i in frstprt if i in scndprt]
        return result

    elif " OR NOT " in query:
        word1, word2 = query.split("OR NOT")
        word1 = preprocess_text(word1)[0]
        word2 = preprocess_text(word2)[0]

        results1 = index[word1]
        results2 = index[word2]

        results1_ids = []
        for i in results1:
            results1_ids.append(i)

        notresult2 = []
        # for i in range(1, index.doc_count + 1):
        for i in Document.objects.values_list('doc_id')[:50]:
            # The doc ids in the index are strings
            # but range gives us integers
            i = str(i[0])
            if i not in results2:
                notresult2.append(i)

        results1_ids = [i for i in notresult2 if i not in results1_ids]
        return results1_ids

    elif 'OR' in query:
        i = 0
        while i < len(spl):
            if spl[i] == "OR":
                i += 1
                break
            frstprt.append(spl[i])
            i += 1
        while i < len(spl):

            if spl[i] == "OR":
                i += 1
                break
            scndprt.append(spl[i])
            i += 1

        frstprt = listToString(frstprt)
        frstprt = preprocess_text(frstprt)[0]
        scndprt = listToString(scndprt)
        scndprt = preprocess_text(scndprt)[0]
        res1 = index[frstprt]
        res2 = index[scndprt]
        res1.update(res2)

        return res1

    elif query.startswith('"') and query.endswith('"'):
        word1,word2 = query.split(" ")

        word1 = preprocess_text(word1)[0]
        word2 = preprocess_text(word2)[0]

        results = set()
        res1 = index[word1]

        for id in res1:
            pos1 = index[word1][id]
            pos2 = index[word2][id]
            locResults = [i for i in pos1 if i + 1 in pos2]
            if locResults:
                results.add(id)
        return results

    elif re.match(match, query):

        m = re.match(match, query)
        num = int(m.group(1))
        word1 = m.group(2)
        word2 = m.group(3)
        word1 = preprocess_text(word1)[0]
        word2 = preprocess_text(word2)[0]

        result = set()
        res1 = index[word1]

        for id in res1:
            cur = 0
            while cur <= num:
                pos1 = index[word1][id]
                pos2 = index[word2][id]
                locResults = [i for i in pos1 if (i + cur) in pos2]
                if locResults:
                    result.add(id)
                cur += 1
            cur = 0
            while cur <= num:
                pos1 = index[word1][id]
                pos2 = index[word2][id]
                locResults = [i for i in pos1 if (i - cur) in pos2]
                if locResults:
                    result.add(id)
                cur += 1
        return result
    else:
        return run_ranked_search(query)