import torch
import re
import pandas as pd
import numpy as np
import requests
import copy
import gensim

from tqdm import tqdm
from sklearn.preprocessing import LabelEncoder

import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F

import warnings
warnings.filterwarnings(action='ignore')

def load_dataset(data_key):
    if data_key == 'news':
        news_dataset = pd.read_excel('./dataset/news_dataset_preprocessing.xlsx')
        del news_dataset['Unnamed: 0']

        lb_encoder = LabelEncoder()
        news_dataset["encode_category"] = lb_encoder.fit_transform(news_dataset["category"])
        news_dataset[["category", "encode_category"]].head()

        return news_dataset
    elif data_key == 'tweet_disaster':
        #tweet_disaster_df = pd.read_csv('./dataset/train_tweet_disaster.csv')
        tweet_disaster_df = pd.read_csv('./dataset/train_refined.csv')
        return tweet_disaster_df
    else:
        print("DATA KEY ERROR: ", data_key)
        return None

def elapsed_time(start, end):
  from datetime import datetime

  fmt = '%Y-%m-%d %H:%M:%S'
  time_start = datetime.strptime(start, fmt)
  time_end = datetime.strptime(end, fmt)

  diff = time_end - time_start

  h = int(diff.seconds/3600)
  m = int((diff.seconds%3600)/60)
  s = (diff.seconds%3600)%60

  print("elapsed time: {:1d}days {:2d}:{:2d}:{:2d}".format(diff.days, h, m, s))

def get_timestamp():
  from datetime import datetime

  fmt = '%Y-%m-%d %H:%M:%S'
  return datetime.now().strftime(fmt)

def get_BERT_vector(sentence_list):
    from transformers import BertModel, BertTokenizer

    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    model = BertModel.from_pretrained('bert-base-uncased')

    X_concat = torch.tensor(0)
    for i, paragraph in enumerate(tqdm(sentence_list, desc='BERT embedding', mininterval=1)):
        input_ids = torch.tensor([tokenizer.encode(paragraph, add_special_tokens=True)])
        with torch.no_grad():
            outputs = model(input_ids)
        last_hidden_states = outputs[0]  # [batch, tokens, hidden_dim (768)]

        mean = torch.mean(last_hidden_states, dim=1)  # tokens
        max, _ = torch.max(last_hidden_states, dim=1)  # tokens
        concat = torch.cat((mean, max), dim=1)
        if i == 0:
            X_concat = copy.deepcopy(concat)
        else:
            X_concat = torch.cat((X_concat, concat), dim=0)
    return X_concat

def get_embedding_vector(KEY, sentence_list):
    eliminated_row = []
    total_eliminated_row = 0
    X_concat = torch.tensor(0)
    for i, sent in enumerate(tqdm(sentence_list, desc=KEY+' embedding', mininterval=1)):
        tokens = sent.split()
        vectors = []
        if KEY == 'new-embedding':
            for token in tokens:
                if token in new_word_lower:
                    # when new word, get embedding from new-embedding model
                    vector = new_embedding_dict.get(token)
                #elif token in convert_word_lower:
                #    vector = convert_embedding_dict.get(token)
                else:
                    # when not new word, get just conceptnet embedding
                    vector = embedding_dict.get(token)

                if vector is not None:
                    vector = np.fromstring(vector, dtype=float, sep=' ')  # when return: embedding
                    vectors.append(vector)
        elif KEY == 'conceptnet':
            #sent = re.sub('\[|\]|:|(|)|\*|#|@|!|\?|\.', '', sent)
            #tokens = sent.lower().split()
            #total = 0
            for token in tokens:
                if token in new_word_lower:
                    # when new word, don't get embedding from conceptnet
                    vector = None
                else:
                    # when not new word, get conceptnet embedding
                    vector = embedding_dict.get(token)

                if vector is not None:
                    #total += 1
                    vector = np.fromstring(vector, dtype=float, sep=' ')  # when return: embedding
                    vectors.append(vector)
        elif KEY == 'glove':
            for token in tokens:
                if token in new_word_lower:
                    vector = None
                else:
                    vector = glove_embedding_dict.get(token)

                if vector is not None:
                    vectors.append(vector)
        elif KEY == 'word2vec':
            for token in tokens:
                if token in new_word_lower:
                    vector = None
                else:
                    if token in word2vec_model:
                        vector = word2vec_model[token]
                    else:
                        vector = None

                if vector is not None:
                    vectors.append(vector)
        else:
            print("KEY Error: ", KEY)
            break

        if vectors == []:
            eliminated_row.append(i)
            total_eliminated_row += 1
        else:
            tensor_vec = torch.tensor(vectors)
            mean = torch.mean(tensor_vec, dim=0)  # tokens
            max, _ = torch.max(tensor_vec, dim=0)  # tokens
            concat = torch.cat((mean, max), dim=0)
            concat = concat.unsqueeze_(dim=0) # 맨앞에 차원 하나 추가 [1, 600]
            if i == 0:
                X_concat = copy.deepcopy(concat)
            else:
                X_concat = torch.cat((X_concat, concat), dim=0)

    """
    if KEY == 'new-embedding':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_newEmbedding.csv', index=False, header=False)
    elif KEY == 'conceptnet':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_conceptnet.csv', index=False, header=False)
    elif KEY == 'glove':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_glove.csv', index=False, header=False)
    elif KEY == 'word2vec':
        df_vec = pd.DataFrame({'X_vector': X_vector})
        df_vec.to_csv('X_vector_word2vec.csv', index=False, header=False)
    else:
        print("KEY Error: ", KEY)
    """

    print("total eliminated row: {}\n".format(total_eliminated_row))
    return X_concat, eliminated_row

def reset_category_list(category_list, eliminated_row):
    for index in sorted(eliminated_row, reverse=True):
        del category_list[index]
    return category_list

##########################################################################
def get_input_dataset(KEY, sentence_list, category_list):
    if KEY == 'bert':
        X_concat = get_BERT_vector(sentence_list)
        X_train, X_test, y_train, y_test = get_train_test_split(X_concat, category_list)
        train_loader, test_loader = get_train_test_loader(X_train, X_test, y_train, y_test)
        return train_loader, test_loader
    else:
        X_concat, eliminated_row = get_embedding_vector(KEY, sentence_list)
        category_list = reset_category_list(category_list, eliminated_row)
        X_train, X_test, y_train, y_test = get_train_test_split(X_concat, category_list)
        train_loader, test_loader = get_train_test_loader(X_train, X_test, y_train, y_test)
        return train_loader, test_loader
##########################################################################

def get_train_test_split(X_concat, category_list):
  from sklearn.model_selection import train_test_split

  y = torch.tensor(category_list)
  X_train, X_test, y_train, y_test = train_test_split(X_concat, y,
                                                      train_size=0.8,
                                                      random_state=928,
                                                      shuffle=True,
                                                      stratify=y)
  return X_train, X_test, y_train, y_test

def get_train_test_loader(X_train, X_test, y_train, y_test):
  from torch.utils.data import TensorDataset, DataLoader

  train_loader = DataLoader(TensorDataset(X_train, y_train),
                            batch_size = B,
                            shuffle = True,
                            num_workers = 2)
  test_loader = DataLoader(TensorDataset(X_test, y_test),
                          batch_size = B,
                          shuffle = True,
                          num_workers = 2)
  return train_loader, test_loader

def flat_accuracy(preds, labels):
  pred_flat = np.argmax(preds, axis=1).flatten()
  labels_flat = labels.flatten()
  return np.sum(pred_flat == labels_flat) / len(labels_flat)

def init_all_random(SET_SEED, r_seed=0, np_seed=0, torch_seed=0, cuda_seed=0):
  import random
  if SET_SEED:
      random.seed(r_seed)
      np.random.seed(np_seed)
      torch.manual_seed(torch_seed)
      torch.cuda.manual_seed_all(cuda_seed)
      torch.backends.cudnn.deterministic = True
      torch.backends.cudnn.benchmark = False
  else:
      print("random\n", random.getstate()[1][0])
      print("numpy.random\n", np.random.get_state()[0], np.random.get_state()[1][0])
      print("torch\n", torch.random.initial_seed())
      print("torch.cuda\n", torch.cuda.random.initial_seed())

def get_model(D_in, H, D_out, activation_fn='ReLU'):
    if activation_fn == 'ReLU6':
        model = nn.Sequential(nn.Linear(D_in, H),
                              nn.ReLU6(),  # ReLU ReLU6 RReLU
                              nn.Linear(H, D_out), )
    elif activation_fn == 'RReLU':
        model = nn.Sequential(nn.Linear(D_in, H),
                              nn.RReLU(),  # ReLU ReLU6 RReLU
                              nn.Linear(H, D_out), )
    else:
        model = nn.Sequential(nn.Linear(D_in, H),
                              nn.ReLU(),  # ReLU ReLU6 RReLU
                              nn.Linear(H, D_out), )

    if USE_CUDA:
        model = model.cuda()
    return model

def run_model(model, lr, loss_fn, optimizer, train_loader, test_loader):
  activation_fn = list(model.children())[1]

  print(f"\nEMBEDDING MODEL: {model_name[KEY]}")
  print(f"learning rate: {lr}\nloss function: {loss_fn.children}")
  print(f"Activation function: {activation_fn}\noptimizer: {optimizer}\n")
  best_accuracy = 0
  best_trn_accuracy = 0
  for epoch in range(500):
    trn_loss, trn_accuracy = [], []
    for X, y in train_loader:
      X = X.float()
      y = y.float()

      if USE_CUDA:
        X = X.cuda()
        y = y.cuda()

      model.train()
      model.zero_grad()

      logits = model(X)  # [batch, D_out] = [32, 4]
      logits_ = F.softmax(logits, dim=1)  # logits_ : [batch, D_out]
      y_pred, _ = torch.max(logits_, dim=1)  # y_pred : [batch]

      loss = loss_fn(y_pred, y)

      pred = logits_.cpu().detach().numpy()
      gt = y.view(-1, 1).cpu().detach().numpy()
      accuracy = flat_accuracy(pred, gt)

      trn_loss.append(loss.item())
      trn_accuracy.append(accuracy)  # for mean accuracy every batch

      loss.backward()
      optimizer.step()

      with torch.no_grad():
        model.eval()

        eval_loss, eval_accuracy = [], []
        for eval_X, eval_y in test_loader:
          eval_X = eval_X.float()
          eval_y = eval_y.float()

          if USE_CUDA:
            eval_X = eval_X.cuda()
            eval_y = eval_y.cuda()

          logits = model(eval_X)
          logits_ = F.softmax(logits, dim=1)
          y_pred, _ = torch.max(logits_, dim=1)

          loss = loss_fn(y_pred, eval_y)

          pred = logits_.cpu().detach().numpy()
          gt = eval_y.view(-1, 1).cpu().detach().numpy()
          accuracy = flat_accuracy(pred, gt)

          eval_loss.append(loss.item())
          eval_accuracy.append(accuracy)
        avg_loss = np.mean(eval_loss, axis=0)
        avg_accuracy = np.mean(eval_accuracy, axis=0)

        if best_accuracy < avg_accuracy:
          best_accuracy = avg_accuracy
          #torch.save(model.state_dict(), model_path[KEY])

    trn_avg_loss = np.mean(trn_loss, axis=0)
    trn_avg_accuracy = np.mean(trn_accuracy, axis=0)
    if best_trn_accuracy < accuracy:
        best_trn_accuracy = accuracy
    if (epoch+1) % 50 == 0:
      print("{:3d}-epoch | trn loss: {:2.2f} | trn best accuracy: {:.4f} | val loss: {:2.2f} | val best accuracy: {:.4f}".format(epoch+1, trn_avg_loss, accuracy, avg_loss, best_accuracy))
  print("BEST TRN ACCURACY: {:.4f}".format(best_trn_accuracy))
  print("BEST VAL ACCURACY: {:.4f}".format(best_accuracy))
  print("\n#####################################\n")
  return best_accuracy

if __name__ == '__main__':
    #data_df = load_dataset('news')
    data_df = load_dataset('tweet_disaster')
    #####################################
    USE_CUDA = False
    if torch.cuda.is_available():
        USE_CUDA = True
    DEVICE = torch.device("cuda:0" if USE_CUDA else "cpu")
    #####################################

    model_input = {"bert": 1536,
                   "conceptnet": 600,
                   "new-embedding": 600,
                   "glove": 600,
                   "word2vec": 600,}
    model_name = {"bert": "BERT Base Uncased",
                  "conceptnet": "ConceptNet",
                  "new-embedding": "New Embedding",
                  "glove": "Glove",
                  "word2vec": "Word2Vec"}
    model_path = {"bert": "best-bert-model.pt",
                  "conceptnet": "best-conceptnet-model.pt",
                  "new-embedding": "best-new-embedding-model.pt",
                  "glove": "best-glove-model.pt",
                  "word2vec": 'best-word2vec-model.pt'}

    #####################################
    # Make embedding dictionary
    embedding_list = pd.read_csv('wordEmbedding_en.csv', encoding='utf-8', usecols=['word', 'vec300'], sep=' ')
    embedding_dict = embedding_list.set_index('word')['vec300'].to_dict()

    # --------------------------------------------------------
    # new word embedding (prediction)
    new_embedding_list = pd.read_csv('prediction_embedding.csv')

    new_word_lower = []
    for w in new_embedding_list['new_word']:
        new_word_lower.append(w.lower())
    new_embedding_list['new_word_lower'] = new_word_lower

    # DataFrame to dictionary
    new_embedding_dict = new_embedding_list.set_index('new_word_lower')['embedding'].to_dict()

    #------------------------------------------------------------------------------------------------------------------
    # disaster tweets data: conceptnet
    for i in range(1):
        KEY = 'conceptnet'

        init_all_random(SET_SEED=True,
                        r_seed=2147483648,
                        np_seed=2147483648,
                        torch_seed=17043145292842172883 & ((1<<63)-1),
                        cuda_seed=5039337088303087)
        D_in = model_input[KEY]
        B, H, D_out = 32, 100, 2

        train_loader, test_loader = get_input_dataset(KEY, data_df['refine_text'], data_df['target'])
        #train_loader, test_loader = get_input_dataset(KEY, data_df['keyword_text'], data_df['target'].tolist())

        model = get_model(D_in, H, D_out, 'ReLU')
        loss_fn = nn.MSELoss(reduction='sum')
        optimizer = optim.Adam(model.parameters(), lr=3e-3)
        time_start = get_timestamp()
        best_accuracy = run_model(model, 3e-3, loss_fn, optimizer, train_loader, test_loader)
        time_end = get_timestamp()
        elapsed_time(start=time_start, end=time_end)
        print('\n\n')


#!/usr/bin/env python3
"""

Copyright (c) 2019-2021 Moein Khazraee

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

import os
import sys

import scapy.utils
from scapy.layers.l2 import Ether
from scapy.layers.inet import IP, UDP

import cocotb_test.simulator

import cocotb
from cocotb.triggers import RisingEdge, Timer

from cocotbext.axi.utils import hexdump_str

from Rosebud_API import TB
import mqnic

FIRMWARE = os.path.abspath(os.path.join(os.path.dirname(__file__),
    '..', '..', '..', '..', '..', 'riscv_code', 'basic_corundum_tb.elf'))

@cocotb.test()
async def run_test_nic(dut):

    tb = TB(dut)

    await tb.init()

    await tb.load_firmware(FIRMWARE)

    tb.log.info("Set core enable mask")
    await tb.set_enable_cores(0xffff)
    await tb.set_receive_cores(0xffff)
    await tb.set_enable_interfaces(0xffff)
    await Timer(100, 'ns')

    tb.log.info("Init driver")
    await tb.driver.init_dev(tb.dev.functions[0].pcie_id)
    await tb.driver.interfaces[0].open()

    # enable queues
    tb.log.info("Enable queues")
    await tb.rc.mem_write_dword(tb.driver.interfaces[0].ports[0].hw_addr+mqnic.MQNIC_PORT_REG_SCHED_ENABLE, 0x00000001)
    for k in range(tb.driver.interfaces[0].tx_queue_count):
        await tb.rc.mem_write_dword(tb.driver.interfaces[0].ports[0].schedulers[0].hw_addr+4*k, 0x00000003)

    # wait for all writes to complete
    await tb.rc.mem_read(tb.driver.hw_addr, 4)
    tb.log.info("Init complete")

    tb.log.info("Send and receive single packet")

    data = bytearray([x % 256 for x in range(1024)])

    await tb.driver.interfaces[0].start_xmit(data, 0)

    pkt = await tb.qsfp0_sink.recv()
    tb.log.info("Packet: %s", pkt)

    await tb.qsfp0_source.send(pkt)

    pkt = await tb.driver.interfaces[0].recv()

    tb.log.info("Packet: %s", pkt)
    assert pkt.rx_checksum == ~scapy.utils.checksum(bytes(pkt.data[14:])) & 0xffff

    tb.log.info("RX and TX checksum tests")

    payload = bytes([x % 256 for x in range(256)])
    eth = Ether(src='5A:51:52:53:54:55', dst='DA:D1:D2:D3:D4:D5')
    ip = IP(src='192.168.1.100', dst='192.168.1.101')
    udp = UDP(sport=1, dport=2)
    test_pkt = eth / ip / udp / payload

    test_pkt2 = test_pkt.copy()
    test_pkt2[UDP].chksum = scapy.utils.checksum(bytes(test_pkt2[UDP]))

    await tb.driver.interfaces[0].start_xmit(test_pkt2.build(), 0, 34, 6)

    pkt = await tb.qsfp0_sink.recv()
    tb.log.info("Packet: %s", pkt)

    await tb.qsfp0_source.send(pkt)

    pkt = await tb.driver.interfaces[0].recv()

    tb.log.info("Packet: %s", pkt)
    assert pkt.rx_checksum == ~scapy.utils.checksum(bytes(pkt.data[14:])) & 0xffff
    assert Ether(pkt.data).build() == test_pkt.build()

    tb.log.info("Multiple small packets")

    count = 64

    pkts = [bytes([(x+k) % 256 for x in range(60)]) for k in range(count)]
    pkts_set = set(pkts)

    tb.loopback_enable = True

    for p in pkts:
        await tb.driver.interfaces[0].start_xmit(p, 0)

    for k in range(count):
        pkt = await tb.driver.interfaces[0].recv()

        tb.log.debug("%s", hexdump_str(bytes(pkt), row_size=32))
        assert pkt.data in pkts_set
        pkts_set.remove(pkt.data)
        assert pkt.rx_checksum == ~scapy.utils.checksum(bytes(pkt.data[14:])) & 0xffff

    tb.loopback_enable = False

    tb.log.info("Multiple large packets")

    count = 64

    pkts = [bytes([(x+k) % 256 for x in range(1514)]) for k in range(count)]
    pkts_set = set(pkts)

    tb.loopback_enable = True

    for p in pkts:
        await tb.driver.interfaces[0].start_xmit(p, 0)

    for k in range(count):
        pkt = await tb.driver.interfaces[0].recv()

        tb.log.debug("%s", hexdump_str(bytes(pkt), row_size=32))
        assert pkt.data in pkts_set
        pkts_set.remove(pkt.data)
        assert pkt.rx_checksum == ~scapy.utils.checksum(bytes(pkt.data[14:])) & 0xffff

    tb.loopback_enable = False

    tb.log.info("Jumbo frames")

    count = 64

    pkts = [bytes([(x+k) % 256 for x in range(9014)]) for k in range(count)]
    pkts_set = set(pkts)

    tb.loopback_enable = True

    for p in pkts:
        await tb.driver.interfaces[0].start_xmit(p, 0)

    for k in range(count):
        pkt = await tb.driver.interfaces[0].recv()

        tb.log.debug("%s", hexdump_str(bytes(pkt), row_size=32))
        assert pkt.data in pkts_set
        pkts_set.remove(pkt.data)
        assert pkt.rx_checksum == ~scapy.utils.checksum(bytes(pkt.data[14:])) & 0xffff

    tb.loopback_enable = False

    await RisingEdge(dut.pcie_clk)
    await RisingEdge(dut.pcie_clk)


import json
import os
from tempfile import NamedTemporaryFile
from typing import List, Tuple

import demisto_client
from demisto_client.demisto_api.rest import ApiException
from demisto_sdk.commands.common.constants import (CONTENT_ENTITIES_DIRS,
                                                   CONTENT_ENTITY_UPLOAD_ORDER,
                                                   INTEGRATIONS_DIR, PACKS_DIR,
                                                   SCRIPTS_DIR, FileType)
from demisto_sdk.commands.common.tools import (
    LOG_COLORS, find_type, get_child_directories, get_child_files, get_json,
    get_parent_directory_name, is_path_of_classifier_directory,
    is_path_of_dashboard_directory, is_path_of_incident_field_directory,
    is_path_of_incident_type_directory, is_path_of_integration_directory,
    is_path_of_layout_directory, is_path_of_playbook_directory,
    is_path_of_script_directory, is_path_of_test_playbook_directory,
    is_path_of_widget_directory, print_color, print_error, print_v)
from demisto_sdk.commands.unify.unifier import Unifier
from tabulate import tabulate

class Uploader:
    """Upload a pack specified in self.infile to a remote Cortex XSOAR instance.
        Attributes:
            path (str): The path of a pack / directory / file to upload.
            verbose (bool): Whether to output a detailed response.
            client (DefaultApi): Demisto-SDK client object.
        """

    def __init__(self, input: str, insecure: bool = False, verbose: bool = False):
        self.path = input
        self.log_verbose = verbose
        verify = (not insecure) if insecure else None  # set to None so demisto_client will use env var DEMISTO_VERIFY_SSL
        self.client = demisto_client.configure(verify_ssl=verify)
        self.status_code = 0
        self.successfully_uploaded_files: List[Tuple[str, str]] = []
        self.failed_uploaded_files: List[Tuple[str, str]] = []

    def upload(self):
        """Upload the pack / directory / file to the remote Cortex XSOAR instance.
        """
        print(f"Uploading {self.path} ...")
        parent_dir_name = get_parent_directory_name(self.path)

        if not os.path.exists(self.path):
            print_error(f'Error: Given input path: {self.path} does not exist')
            self.status_code = 1

        # Input is a file
        elif os.path.isfile(self.path):
            file_type = find_type(self.path)
            if file_type == FileType.INTEGRATION:
                self.integration_uploader(self.path)
            elif file_type in (FileType.SCRIPT, FileType.TEST_SCRIPT):
                self.script_uploader(self.path)
            elif file_type in (FileType.PLAYBOOK, FileType.TEST_PLAYBOOK):
                self.playbook_uploader(self.path)
            elif file_type == FileType.WIDGET:
                self.widget_uploader(self.path)
            elif file_type == FileType.INCIDENT_TYPE:
                self.incident_type_uploader(self.path)
            elif file_type == FileType.CLASSIFIER:
                self.classifier_uploader(self.path)
            elif file_type == FileType.OLD_CLASSIFIER:
                self.classifier_uploader(self.path)
            elif file_type == FileType.LAYOUT:
                self.layout_uploader(self.path)
            elif file_type == FileType.LAYOUTS_CONTAINER:
                self.layout_uploader(self.path)
            elif file_type == FileType.DASHBOARD:
                self.dashboard_uploader(self.path)
            elif file_type == FileType.INCIDENT_FIELD:
                self.incident_field_uploader(self.path)
            else:
                print_error(
                    f'\nError: Given input path: {self.path} is not valid. '
                    f'Input path should point to one of the following:\n'
                    f'  1. Pack\n'
                    f'  2. A content entity directory that is inside a pack. For example: an Integrations directory or '
                    f'a Layouts directory\n'
                    f'  3. Valid file that can be imported to Cortex XSOAR manually. '
                    f'For example a playbook: helloWorld.yml'
                )
                self.status_code = 1

        elif os.path.isdir(self.path):
            # Input is an integration directory (HelloWorld)
            if parent_dir_name == INTEGRATIONS_DIR:
                self.integration_uploader(self.path)

            # Input is a script directory (commonServerPython)
            elif parent_dir_name == SCRIPTS_DIR:
                self.script_uploader(self.path)

            # Input is a content entity directory (Integrations/Scripts/Playbook etc...)
            elif os.path.basename(self.path.rstrip('/')) in CONTENT_ENTITIES_DIRS:
                self.directory_uploader(self.path.rstrip('/'))

            # Input is a pack
            elif parent_dir_name == PACKS_DIR:
                self.pack_uploader()

            # Input is not supported
            else:
                print_error(
                    f'\nError: Given input path: {self.path} is not valid. '
                    f'Input path should point to one of the following:\n'
                    f'  1. Pack\n'
                    f'  2. A content entity directory that is inside a pack. For example: an Integrations directory or '
                    f'a Layouts directory\n'
                    f'  3. Valid file that can be imported to Cortex XSOAR manually. '
                    f'For example a playbook: helloWorld.yml'
                )
                self.status_code = 1

        self._print_summary()
        return self.status_code

    def pack_uploader(self):
        """Extracts the directories of the pack and upload them by directory_uploader
        """
        list_directories = get_child_directories(self.path)
        ordered_directories_list = self._sort_directories_based_on_dependencies(list_directories)
        for directory in ordered_directories_list:
            self.directory_uploader(directory)

    def directory_uploader(self, path: str):
        """Uploads directories by path

        Args:
            path (str): Path for directory to upload.
        """
        if is_path_of_integration_directory(path):
            # Upload unified integration files
            list_unified_integrations = get_child_files(path)
            for unified_integration in list_unified_integrations:
                file_type = find_type(unified_integration)
                if file_type == FileType.INTEGRATION:
                    self.integration_uploader(unified_integration)
            # Upload spliced integration files
            list_integrations = get_child_directories(path)
            for integration in list_integrations:
                self.integration_uploader(integration)

        elif is_path_of_script_directory(path):
            # Upload unified scripts files
            list_unified_scripts = get_child_files(path)
            for unified_script in list_unified_scripts:
                file_type = find_type(unified_script)
                if file_type in (FileType.SCRIPT, FileType.TEST_SCRIPT):
                    self.script_uploader(unified_script)
            # Upload spliced scripts
            list_script = get_child_directories(path)
            for script in list_script:
                self.script_uploader(script)

        elif is_path_of_playbook_directory(path) or is_path_of_test_playbook_directory(path):
            list_playbooks = get_child_files(path)
            for playbook in list_playbooks:
                if playbook.endswith('.yml'):
                    self.playbook_uploader(playbook)

        elif is_path_of_incident_field_directory(path):
            list_incident_fields = get_child_files(path)
            for incident_field in list_incident_fields:
                if incident_field.endswith('.json'):
                    self.incident_field_uploader(incident_field)

        elif is_path_of_widget_directory(path):
            list_widgets = get_child_files(path)
            for widget in list_widgets:
                if widget.endswith('.json'):
                    self.widget_uploader(widget)

        elif is_path_of_dashboard_directory(path):
            list_dashboards = get_child_files(path)
            for dashboard in list_dashboards:
                if dashboard.endswith('.json'):
                    self.dashboard_uploader(dashboard)

        elif is_path_of_layout_directory(path):
            list_layouts = get_child_files(path)
            for layout in list_layouts:
                if layout.endswith('.json'):
                    self.layout_uploader(layout)

        elif is_path_of_incident_type_directory(path):
            list_incident_types = get_child_files(path)
            for incident_type in list_incident_types:
                if incident_type.endswith('.json'):
                    self.incident_type_uploader(incident_type)

        elif is_path_of_classifier_directory(path):
            list_classifiers = get_child_files(path)
            for classifiers in list_classifiers:
                if classifiers.endswith('.json'):
                    self.classifier_uploader(classifiers)

    def integration_uploader(self, path: str):
        is_dir = False
        file_name = os.path.basename(path)
        docker45_path = ''

        try:
            if os.path.isdir(path):  # Create a temporary unified yml file
                try:
                    is_dir = True
                    unifier = Unifier(input=path, output=path)
                    unified_paths = unifier.merge_script_package_to_yml()
                    path = unified_paths[0]
                    docker45_path = unified_paths[1] if len(unified_paths) > 1 else ''
                    file_name = os.path.basename(path)
                except IndexError:
                    print_error(f'Error uploading integration from pack. /'
                                f'Check that the given integration path contains a valid integration: {path}.')

                    self.status_code = 1
                    self.failed_uploaded_files.append((file_name, 'Integration'))
                    return

                except Exception as err:
                    print_error(str('Upload integration failed\n'))
                    print_error(str(err))
                    self.failed_uploaded_files.append((file_name, 'Integration'))
                    self.status_code = 1
                    return

            # Upload the file to Cortex XSOAR
            result = self.client.integration_upload(file=path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded integration - \'{file_name}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Integration'))

        except Exception as err:
            self._parse_error_response(err, 'integration', file_name)
            self.failed_uploaded_files.append((file_name, 'Integration'))
            self.status_code = 1

        finally:
            # Remove the temporary file
            if is_dir:
                self._remove_temp_file(path)
                if docker45_path:
                    self._remove_temp_file(docker45_path)

    def script_uploader(self, path: str):
        is_dir = False
        file_name = os.path.basename(path)
        docker45_path = ''

        try:
            if os.path.isdir(path):  # Create a temporary unified yml file
                is_dir = True
                try:
                    unifier = Unifier(input=path, output=path)
                    unified_paths = unifier.merge_script_package_to_yml()
                    path = unified_paths[0]
                    docker45_path = unified_paths[1] if len(unified_paths) > 1 else ''
                    file_name = os.path.basename(path)
                except IndexError:
                    print_error(f'Error uploading script from pack. /'
                                f'Check that the given script path contains a valid script: {path}.')
                    self.status_code = 1
                    self.failed_uploaded_files.append((file_name, 'Script'))
                    return

                except Exception as err:
                    print_error(str('Upload script failed\n'))
                    print_error(str(err))
                    self.status_code = 1
                    self.failed_uploaded_files.append((file_name, 'Script'))
                    return

            # Upload the file to Cortex XSOAR
            result = self.client.import_script(file=path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded script - \'{file_name}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Script'))

        except Exception as err:
            self._parse_error_response(err, 'script', file_name)
            self.failed_uploaded_files.append((file_name, 'Script'))
            self.status_code = 1

        finally:
            # Remove the temporary file
            if is_dir:
                self._remove_temp_file(path)
                if docker45_path:
                    self._remove_temp_file(docker45_path)

    def playbook_uploader(self, path: str):
        file_name = os.path.basename(path)

        try:
            # Upload the file to Cortex XSOAR
            result = self.client.import_playbook(file=path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded playbook - \'{file_name}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Playbook'))

        except Exception as err:
            self._parse_error_response(err, 'playbook', file_name)
            self.failed_uploaded_files.append((file_name, 'Playbook'))
            self.status_code = 1

    def incident_field_uploader(self, path: str):
        file_name = os.path.basename(path)

        try:
            # Wrap the incident object with a list to be compatible with Cortex XSOAR
            incident_fields_unified_data = {'incidentFields': [get_json(path)]}
            # Create a temp file object
            incidents_unified_file = NamedTemporaryFile(dir=f'{os.path.dirname(path)}', suffix='.json', delete=False)
            incidents_unified_file.write(bytes(json.dumps(incident_fields_unified_data), 'utf-8'))
            new_file_path = incidents_unified_file.name
            incidents_unified_file.close()

            # Upload the file to Cortex XSOAR
            result = self.client.import_incident_fields(file=new_file_path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded incident field - \'{os.path.basename(path)}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Incident Field'))

        except Exception as err:
            self._parse_error_response(err, 'incident field', file_name)
            self.failed_uploaded_files.append((file_name, 'Incident Field'))
            self.status_code = 1

        finally:
            self._remove_temp_file(new_file_path)

    def widget_uploader(self, path: str):
        file_name = os.path.basename(path)

        try:
            # Upload the file to Cortex XSOAR
            result = self.client.import_widget(file=path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded widget - \'{os.path.basename(path)}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Widget'))

        except Exception as err:
            self._parse_error_response(err, 'widget', file_name)
            self.failed_uploaded_files.append((file_name, 'Widget'))
            self.status_code = 1

    def dashboard_uploader(self, path: str):
        file_name = os.path.basename(path)

        try:
            # Upload the file to Cortex XSOAR
            result = self.client.import_dashboard(file=path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded dashboard - \'{os.path.basename(path)}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Dashboard'))

        except Exception as err:
            self._parse_error_response(err, 'dashboard', file_name)
            self.failed_uploaded_files.append((file_name, 'Dashboard'))
            self.status_code = 1

    def layout_uploader(self, path: str):
        file_name = os.path.basename(path)

        try:
            # Upload the file to Cortex XSOAR
            result = self.client.import_layout(file=path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded layout - \'{os.path.basename(path)}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Layout'))

        except Exception as err:
            self._parse_error_response(err, 'layout', file_name)
            self.failed_uploaded_files.append((file_name, 'Layout'))
            self.status_code = 1

    def incident_type_uploader(self, path: str):
        file_name = os.path.basename(path)

        try:
            # Wrap the incident object with a list to be compatible with Cortex XSOAR
            incident_types_unified_data = [get_json(path)]
            # Create a temp file object
            incidents_unified_file = NamedTemporaryFile(dir=f'{os.path.dirname(path)}', suffix='.json', delete=False)
            incidents_unified_file.write(bytes(json.dumps(incident_types_unified_data), 'utf-8'))
            new_file_path = incidents_unified_file.name
            incidents_unified_file.close()

            # Upload the file to Cortex XSOAR
            result = self.client.import_incident_types_handler(file=new_file_path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded incident type - \'{os.path.basename(path)}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Incident Type'))

        except Exception as err:
            self._parse_error_response(err, 'incident type', file_name)
            self.failed_uploaded_files.append((file_name, 'Incident Type'))
            self.status_code = 1

        finally:
            self._remove_temp_file(new_file_path)

    def classifier_uploader(self, path: str):
        file_name = os.path.basename(path)

        try:
            # Upload the file to Cortex XSOAR
            result = self.client.import_classifier(file=path)

            # Print results
            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
            print_color(f'Uploaded classifier - \'{os.path.basename(path)}\': successfully', LOG_COLORS.GREEN)
            self.successfully_uploaded_files.append((file_name, 'Classifier'))
        except Exception as err:
            self._parse_error_response(err, 'classifier', file_name)
            self.failed_uploaded_files.append((file_name, 'Classifier'))
            self.status_code = 1

    def _parse_error_response(self, error: ApiException, file_type: str, file_name: str):
        """Parses error message from exception raised in call to client to upload a file

        error (ApiException): The exception which was raised in call in to client
        file_type (str): The file type which was attempted to be uploaded
        file_name (str): The file name which was attempted to be uploaded
        """
        message = error
        if hasattr(error, 'reason'):
            if '[SSL: CERTIFICATE_VERIFY_FAILED]' in str(error.reason):
                message = '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate.\n' \
                          'Try running the command with --insecure flag.'

            elif 'Failed to establish a new connection:' in str(error.reason):
                message = 'Failed to establish a new connection: Connection refused.\n' \
                          'Try checking your BASE url configuration.'

            elif error.reason in ('Bad Request', 'Forbidden'):
                error_body = json.loads(error.body)
                message = error_body.get('error')

                if error_body.get('status') == 403:
                    message += '\nTry checking your API key configuration.'
        print_error(str(f'\nUpload {file_type}: {file_name} failed:'))
        print_error(str(message))

    def _remove_temp_file(self, path_to_delete):
        if os.path.exists(path_to_delete) and not os.path.isdir(path_to_delete):
            try:
                os.unlink(path_to_delete)
            except (PermissionError, IsADirectoryError) as error:
                print_error(error)

    def _sort_directories_based_on_dependencies(self, dir_list: List) -> List:
        """Sorts given list of directories based on logic order of content entities that depend on each other.
        If a given directory does not appear in the CONTENT_ENTITY_UPLOAD_ORDER list it will be ignored

        Args:
            dir_list (List): List of directories to sort

        Returns:
            List. The sorted list of directories.
        """
        srt = {item: index for index, item in enumerate(CONTENT_ENTITY_UPLOAD_ORDER)}
        for dir_path in dir_list:
            if os.path.basename(dir_path) not in CONTENT_ENTITY_UPLOAD_ORDER:
                dir_list.remove(dir_path)
        dir_list.sort(key=lambda item: srt.get(os.path.basename(item)))
        return dir_list

    def _print_summary(self):
        """Prints uploaded files summary
        Successful uploads grid based on `successfully_uploaded_files` attribute in green color
        Failed uploads grid based on `failed_uploaded_files` attribute in red color
        """
        print_color('\n\nUPLOAD SUMMARY:', LOG_COLORS.NATIVE)
        if self.successfully_uploaded_files:
            print_color('\nSUCCESSFUL UPLOADS:', LOG_COLORS.GREEN)
            print_color(tabulate(self.successfully_uploaded_files, headers=['NAME', 'TYPE'],
                                 tablefmt="fancy_grid") + '\n', LOG_COLORS.GREEN)
        if self.failed_uploaded_files:
            print_color('\nFAILED UPLOADS:', LOG_COLORS.RED)
            print_color(tabulate(self.failed_uploaded_files, headers=['NAME', 'TYPE'],
                                 tablefmt="fancy_grid") + '\n', LOG_COLORS.RED)


# here I will be argeparse module to use command line arguments and interact with my program..

import sys
import argparse

def game():
    parser = argparse.ArgumentParser()
    parser.add_argument("--x", type=int)
    parser.add_argument("--y",type=int)
    parser.add_argument("--operation",type=str)

    # now we have to make parsed arguments
    argo = parser.parse_args()
    sys.stdout.write(str(calc(argo)))

def calc(argo):
    if argo.operation == "add":
        return argo.x + argo.y
    if argo.operation == "sub":
        return argo.x - argo.y
    if argo.operation == "mul":
        return argo.x * argo.y
    if argo.operation == "div":
        return argo.x // argo.y

game()


#!/usr/bin/env python
# coding: utf-8

import sys
import pyspark
import string

from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark.sql import SQLContext

from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType
from pyspark.sql.window import Window
from pyspark.sql.functions import *

if __name__ == "__main__":
    sc = SparkContext()

    spark = SparkSession \
        .builder \
        .appName("hw2sql") \
        .config("spark.some.config.option", "some-value") \
        .getOrCreate()

    sqlContext = SQLContext(spark)

    # get command-line arguments
    inFile = sys.argv[1]
    supp = sys.argv[2]
    conf = sys.argv[3]
    prot = sys.argv[4]

    print(
        "Executing HW2SQL with input from " + inFile + ", support=" + supp + ", confidence=" + conf + ", protection=" + prot)

    pp_schema = StructType([
        StructField("uid", IntegerType(), True),
        StructField("attr", StringType(), True),
        StructField("val", IntegerType(), True)])

    Pro_Publica = sqlContext.read.format('csv').options(header=False).schema(pp_schema).load(inFile)
    Pro_Publica.createOrReplaceTempView("Pro_Publica")
    sqlContext.cacheTable("Pro_Publica")
    spark.sql("select count(*) from Pro_Publica").show()

    # compute frequent itemsets of size 1, store in F1(attr, val)
    query = "select attr, val, count(*) as supp \
               from Pro_Publica \
              group by attr, val \
             having count(*) >= " + str(supp);
    F1 = spark.sql(query);
    F1.createOrReplaceTempView("F1")

    # YOUR SparkSQL CODE GOES HERE
    # You may use any valid SQL query, and store the output in intermediate temporary views
    # Output must go into R2, R3 and PD_R3 as stated below.  Do not change the format of the output
    # on the last three lines.

    # Compute R2, as described in the homework specification
    # R2(attr1, val1, attr2, val2, supp, conf)

    query = "SELECT * FROM (select iter.attr as attr1, iter.val as val1, vdectable.attr as attr2, vdectable.val as val2, count(*) as supportspecific " \
            "FROM (select uid, attr, val from Pro_Publica where attr<>'vdecile') iter LEFT JOIN (SELECT * FROM Pro_Publica WHERE attr='vdecile') vdectable " \
            "where iter.uid=vdectable.uid GROUP BY iter.attr, iter.val, vdectable.attr, vdectable.val) inti WHERE inti.supportspecific>500"
    F2 = spark.sql(query);
    F2.createOrReplaceTempView("F2")

    query = "select V.attr1 , V.val1 ,V.attr2, V.val2, V.SUPPORTSPECIFIC as supp,CAST(((V.SUPPORTSPECIFIC * 1.0)/E.supp) AS FLOAT) as  conf from F1 E inner join F2 V ON (E.attr=V.attr1 and E.val=V.val1) WHERE (V.SUPPORTSPECIFIC >" + str(
        supp) + ") and ((V.SUPPORTSPECIFIC * 1.0)/E.supp) > " + str(
        conf) + " ORDER BY V.attr1, V.val1, V.attr2 ,V.val2";
    R2 = spark.sql(query)
    R2.createOrReplaceTempView("R2")

    # MORE OF YOUR SparkSQL CODE GOES HERE

    # Compute R3, as described in the homework specification
    # R3(attr1, val1, attr2, val2, attr3, val3, supp, conf)

    totalsum = "SELECT * FROM (SELECT itable.attr1 , itable.val1 ,itable.attr2, itable.val2, count(*) AS SUPPORTPARTIAL FROM (SELECT A.uid AS uniqueid, A.attr AS attr1, A.val AS val1, B.attr AS attr2, B.val AS val2 FROM Pro_Publica A, Pro_Publica B WHERE A.attr <> B.attr AND A.uid = B.uid AND A.attr <> 'vdecile' AND B.attr <> 'vdecile') itable WHERE itable.attr1 < itable.attr2 GROUP BY itable.attr1 , itable.val1 ,itable.attr2, itable.val2) ink WHERE ink.SUPPORTPARTIAL>500"

    UI = spark.sql(totalsum);
    UI.createOrReplaceTempView("UI")

    specificsup = "SELECT * FROM (SELECT itable.attr1 , itable.val1 ,itable.attr2, itable.val2, vdectable.attr, vdectable.val, count(*) AS SUPPORTSPECIFIC FROM (SELECT A.uid AS uniqueid, A.attr AS attr1, A.val AS val1, B.attr AS attr2, B.val AS val2 FROM Pro_Publica A, Pro_Publica B WHERE A.attr <> B.attr AND A.uid = B.uid AND A.attr <> 'vdecile' AND B.attr <> 'vdecile') itable LEFT JOIN (SELECT * FROM Pro_Publica WHERE attr='vdecile') vdectable ON (itable.uniqueid = vdectable.uid) WHERE itable.attr1 < itable.attr2 GROUP BY itable.attr1 , itable.val1 ,itable.attr2, itable.val2, vdectable.attr, vdectable.val) inka WHERE inka.SUPPORTSPECIFIC>500"
    UIS = spark.sql(specificsup);
    UIS.createOrReplaceTempView("UIS")

    query = "select E.attr1 , E.val1 ,E.attr2, E.val2, V.attr as attr3,V.val as val3, V.SUPPORTSPECIFIC as supp,CAST(((V.SUPPORTSPECIFIC * 1.0)/E.SUPPORTPARTIAL) AS FLOAT) as  conf from UI E inner join UIS V ON (E.attr1=V.attr1 and E.val1=V.val1 and E.attr2=V.attr2 and E.val2=V.val2) WHERE (V.SUPPORTSPECIFIC >" + str(
        supp) + ") and ((V.SUPPORTSPECIFIC * 1.0)/E.SUPPORTPARTIAL) > " + str(
        conf) + " ORDER BY E.attr1, E.val1, E.attr2 ,E.val2, V.attr, V.val";
    R3 = spark.sql(query)
    R3.createOrReplaceTempView("R3")
    R3.show()

    # MORE OF YOUR SparkSQL CODE GOES HERE

    # Compute PD_R3, as described in the homework specification
    # PD_R3(attr1, val1, attr2, val2, attr3, val3, supp, conf, prot)
    query = "SELECT Distinct E.attr1 , E.val1 ,E.attr2, E.val2, E.attr3,E.val3, E.supp, E.conf, CAST(round(((E.conf * 1.0)/U.conf),2) AS FLOAT) as prot from R3 E,R2 U,R2 V WHERE (V.attr1=E.attr2 OR V.attr1=E.attr1) AND V.attr1='race' AND ( CASE WHEN E.attr2='race' THEN (E.attr1=U.attr1 AND E.val1=U.val1) WHEN E.attr1='race' THEN (E.attr2=U.attr2 AND E.val2=U.val2) END ) AND ((E.conf * 1.0)/U.conf)>" + str(
        prot) + " ORDER BY E.attr1, E.val1, E.attr2 ,E.val2, E.attr3, E.val3";

    PD_R3 = spark.sql(query)
    PD_R3.createOrReplaceTempView("PD_R3")

    R2.select(format_string('%s,%s,%s,%s,%d,%.2f', R2.attr1, R2.val1, R2.attr2, R2.val2, R2.supp,
                            R2.conf)).write.save("r2.out", format="text")
    R3.select(
        format_string('%s,%s,%s,%s,%s,%s,%d,%.2f', R3.attr1, R3.val1, R3.attr2, R3.val2, R3.attr3,
                      R3.val3, R3.supp, R3.conf)).write.save("r3.out", format="text")
    PD_R3.select(
        format_string('%s,%s,%s,%s,%s,%s,%d,%.2f,%.2f', PD_R3.attr1, PD_R3.val1, PD_R3.attr2,
                      PD_R3.val2, PD_R3.attr3, PD_R3.val3, PD_R3.supp, PD_R3.conf,
                      PD_R3.prot)).write.save("pd-r3.out", format="text")

    sc.stop()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 11 11:02:21 2018

@author: qifeng
"""

# coding: utf-8

# In[1]:

import cv2
import pandas as pd
import numpy as np
import os
import pickle
from collections import defaultdict
from scipy.stats import itemfreq
from scipy import ndimage as ndi
import matplotlib.pyplot as plt
from skimage import feature
import operator
from PIL import Image as IMG
from tqdm import tqdm

# In[7]:

def bin_spatial(img, size = (32,32)):
    path = img
    img1 = cv2.imread(path)
    img2 = cv2.resize(img1, size)
    return img2

def cs_conversion(img, cspace=['HSV', 'LUV', 'HLS', 'YUV']):
        feature_HSV = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        feature_LUV = cv2.cvtColor(img, cv2.COLOR_BGR2LUV)
        feature_HLS = cv2.cvtColor(img, cv2.COLOR_BGR2HLS)
        feature_YUV = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
        return feature_HSV, feature_LUV, feature_HLS, feature_YUV

def get_average_color(img):
    average_color = [img[:, :, i].mean() for i in range(img.shape[-1])]
    return average_color

def get_features(img_names):

    average_HSV_Hs = []
    average_HSV_Ss = []
    average_HSV_Vs = []
    average_LUV_Ls = []
    average_LUV_Us = []
    average_LUV_Vs = []
    average_HLS_Hs = []
    average_HLS_Ls = []
    average_HLS_Ss = []
    average_YUV_Ys = []
    average_YUV_Us = []
    average_YUV_Vs = []

    ids = []

    for img_name in tqdm(img_names):
        img_path = image_dir + img_name
        try:
            id_ = img_name[:-4]
            img = cv2.imread(img_path)
#            img = bin_spatial(img_path, size=(32,32))
            feature_HSV, feature_LUV, feature_HLS, feature_YUV = cs_conversion(img, cspace=['HSV', 'LUV', 'HLS', 'YUV'])

            average_HSV = get_average_color(feature_HSV)
            average_LUV = get_average_color(feature_LUV)
            average_HLS = get_average_color(feature_HLS)
            average_YUV = get_average_color(feature_YUV)

            average_HSV_H = average_HSV[0]/ 255.
            average_HSV_S = average_HSV[1]/ 255.
            average_HSV_V = average_HSV[2]/ 255.
            average_LUV_L = average_LUV[0]/ 255.
            average_LUV_U = average_LUV[1]/ 255.
            average_LUV_V = average_LUV[2]/ 255.
            average_HLS_H = average_HLS[0]/ 255.
            average_HLS_L = average_HLS[1]/ 255.
            average_HLS_S = average_HLS[2]/ 255.
            average_YUV_Y = average_YUV[0]/ 255.
            average_YUV_U = average_YUV[1]/ 255.
            average_YUV_V = average_YUV[2]/ 255.

        except:
            continue

        average_HSV_Hs.append(average_HSV_H)
        average_HSV_Ss.append(average_HSV_S)
        average_HSV_Vs.append(average_HSV_V)
        average_LUV_Ls.append(average_LUV_L)
        average_LUV_Us.append(average_LUV_U)
        average_LUV_Vs.append(average_LUV_V)
        average_HLS_Hs.append(average_HLS_H)
        average_HLS_Ls.append(average_HLS_L)
        average_HLS_Ss.append(average_HLS_S)
        average_YUV_Ys.append(average_YUV_Y)
        average_YUV_Us.append(average_YUV_U)
        average_YUV_Vs.append(average_YUV_V)
        ids.append(id_)

    return ids, average_HSV_Hs, average_HSV_Ss, average_HSV_Vs, average_LUV_Ls, average_LUV_Us, average_LUV_Vs, average_HLS_Hs, average_HLS_Ls, average_HLS_Ss, average_YUV_Ys, average_YUV_Us, average_YUV_Vs

# def paralleize(img_names):
#     img_names_split = np.array_split(img_names)
#     pool = Pool(20)
#     feature_list = pool.map(get_features, img_names_split)
#     for feature_index in range(len(feature_list[0])):
#         np.[feature_list[list_index][feature_index] for list_index in range(len(feature_list))]

image_dir = 'input/train_jpg/data/competition_files/train_jpg/'
print('getting train img names')

# In[5]:

img_names = [i for i in os.walk(image_dir)][0][2]

'''
for subdir, dirs, files in os.walk(image_dir):
    for file in files:
        img_names = os.path.join(subdir, file)
'''
print(f'size of images: {len(img_names)}')

ids, average_HSV_Hs, average_HSV_Ss, average_HSV_Vs, average_LUV_Ls, average_LUV_Us, average_LUV_Vs, average_HLS_Hs, average_HLS_Ls, average_HLS_Ss, average_YUV_Ys, average_YUV_Us, average_YUV_Vs = get_features(img_names)

    # print(feature.shape)
cspace = {}

cspace['average_HSV_Hs'] = average_HSV_Hs
cspace['average_HSV_Ss'] = average_HSV_Ss
cspace['average_HSV_Vs'] = average_HSV_Vs
cspace['average_LUV_Ls'] = average_LUV_Ls
cspace['average_LUV_Us'] = average_LUV_Us
cspace['average_LUV_Vs'] = average_LUV_Vs
cspace['average_HLS_Hs'] = average_HLS_Hs
cspace['average_HLS_Ls'] = average_HLS_Ls
cspace['average_HLS_Ss'] = average_HLS_Ss
cspace['average_YUV_Ys'] = average_YUV_Ys
cspace['average_YUV_Us'] = average_YUV_Us
cspace['average_YUV_Vs'] = average_YUV_Vs
cspace['ids'] = ids

with open('input/train_image_features_cspace.p','wb') as f:
    pickle.dump(cspace, f)
del cspace
del ids,\
    average_HSV_Hs,\
    average_HSV_Ss,\
    average_HSV_Vs,\
    average_LUV_Ls,\
    average_LUV_Us,\
    average_LUV_Vs,\
    average_HLS_Hs,\
    average_HLS_Ls,\
    average_HLS_Ss,\
    average_YUV_Ys,\
    average_YUV_Us,\
    average_YUV_Vs
del img_names
import gc
gc.collect()


#A Confederação Nacional de Natação precisa de um programa que leia o ano de nascimento de um atleta e mostre sua categoria, de acordo com a idade:
#- Até 9 anos: MIRIM
#- Até 14 anos: INFANTIL
#- Até 19 anos: JÚNIOR
#- Até 25 anos: SÊNIOR
#- Acima de 25 anos: MASTER
import datetime
anoatual = datetime.datetime.today().year
nascimento = int(input('Digite o seu ano de nascimento: '))
idade = anoatual - nascimento
if anoatual > nascimento:
    if idade <= 9:
        print('Categoria: MIRIM')
    elif idade <= 14:
        print('Categoria: INFANTIL')
    elif idade <= 19:
        print('Categoria: JÚNIOR')
    elif idade <= 25:
        print('Categoria: SÊNIOR')
    else:
        print('Categoria: MASTER')
else:
    print('Erro.Tente outra vez, com valores válidos!')


from django.core.management.base import BaseCommand

from volunteers.emails import VolunteerOutreachChefSignupsEmail

class Command(BaseCommand):
    help = 'Send mass email'

    def handle(self, *args, **options):
        self.stdout.write('Sending volunteer outreach chef signup mass email')
        results = VolunteerOutreachChefSignupsEmail().mass_send()
        self.stdout.write('Sent to:')
        for result in results:
            for email in result.to:
                self.stdout.write(email)


from rest_framework.test import APIClient
from django.core.urlresolvers import reverse
from django.test import TestCase
from .models import Enterprise, District, EnterpriseNetwork, Merchandise, Category
import json

#  TestCase для проверки добавления данных в таблицу
class ModelTest(TestCase):
    def setUp(self):
        District.objects.create(name='District 1')
        self.district = District.objects.get(name='District 1').id

        EnterpriseNetwork.objects.create(name='Enterprise network 1')
        self.enterprise_network = EnterpriseNetwork.objects.get(name='Enterprise network 1').id

        self.enterprise = Enterprise(
            id=1,
            district=[self.enterprise_network],
            name='Enterprise 1',
            description='Description for Enterprise 1',
            affiliation=[self.enterprise_network],
        )

    def test_enterprise_model(self):
        old_count = Enterprise.objects.count()
        self.enterprise.save()
        new_count = Enterprise.objects.count()
        self.assertNotEqual(old_count, new_count)

    def test_merchandise_model(self):
        self.enterprise.save()

        Category.objects.create(name='Category 1')
        self.merchandise = Merchandise(
            id=1,
            name='Merchandise 1',
            category=Category.objects.get(name='Category 1'),
            price=250,
            enterprise=Enterprise.objects.get(name='Enterprise 1'),
            sales_in=[self.enterprise_network]
        )
        old_count_merch = Merchandise.objects.count()
        self.merchandise.save()
        new_count_merch = Merchandise.objects.count()
        self.assertNotEqual(old_count_merch, new_count_merch)

#  TestCase для проверки поиска организаций по району из общего списка районов
#  через URL (organizations/district_<district_id>/...)
class RestAPITest(TestCase):
    def setUp(self):
        self.client = APIClient()

        District.objects.create(name='District 1')
        district = District.objects.get(name='District 1').id

        EnterpriseNetwork.objects.create(name='Enterprise network 1')
        enterprise_network = EnterpriseNetwork.objects.get(name='Enterprise network 1').id

        self.enterprise_name = 'Enterprise 1'
        enterprise_description = 'Description for Enterprise 1'

        Enterprise.objects.create(
            id=1,
            district=[district],
            name=self.enterprise_name,
            description=enterprise_description,
            affiliation=[enterprise_network],
        )

    def test_returning_data(self):
        enterprise = Enterprise.objects.get(name=self.enterprise_name)
        response = self.client.get(
            reverse('organization-district', args=[enterprise.id])
        )
        self.assertEqual(len(response.data), 1)
        self.assertEqual(response.status_code, 200)
        self.assertContains(
            response,
            enterprise
        )

#  TestCase для проверки информации о товаре или услуги
class MerchDetailsTest(TestCase):
    def setUp(self):
        self.client = APIClient()

        District.objects.create(name='District 1')
        self.district = District.objects.get(name='District 1')

        enterprise_name = 'Enterprise 1'
        enterprise_description = 'Description for Enterprise 1'

        EnterpriseNetwork.objects.create(name='Enterprise network 1')
        self.enterprise_network = EnterpriseNetwork.objects.get(name='Enterprise network 1').id

        Category.objects.create(name='Category 1')

        Enterprise.objects.create(
            id=1,
            district=[self.enterprise_network],
            name=enterprise_name,
            description=enterprise_description,
            affiliation=[self.enterprise_network],
        )
        self.enterprise_obj = Enterprise.objects.get(name=enterprise_name)

        self.merch_name = 'Merchandise 1'
        self.merch_category = Category.objects.get(name='Category 1')
        self.merch_price = 250
        self.merch_id = 1

        Merchandise.objects.create(
            id=self.merch_id,
            name=self.merch_name,
            category=self.merch_category,
            price=self.merch_price,
            enterprise=self.enterprise_obj,
            sales_in=[self.enterprise_network]
        )

    def test_merch_details(self):
        merchandise = Merchandise.objects.get(name=self.merch_name)
        response = self.client.get(
            reverse('merchandise-detail', args=[merchandise.id])
        )
        self.assertEqual(response.status_code, 200)
        self.assertDictEqual(response.data, {
            'id': self.merch_id,
            'name': self.merch_name,
            'category': self.merch_category.name,
            'price': self.merch_price,
            'enterprise_id': self.enterprise_obj.id,
            'enterprise_name': self.enterprise_obj.name
        })

#  TestCase для проверки информации об организации
class EnterpriseDetailsTest(TestCase):
    def setUp(self):
        self.client = APIClient()

        EnterpriseNetwork.objects.create(name='Enterprise network 1')
        self.enterprise_network = EnterpriseNetwork.objects.get(name='Enterprise network 1').id
        self.enterprise_name = 'Enterprise 1'
        Enterprise.objects.create(
            id=1,
            district=[self.enterprise_network],
            name='Enterprise 1',
            description='Description for Enterprise 1',
            affiliation=[self.enterprise_network],
        )

    def test_enterprise_details(self):
        enterprise = Enterprise.objects.get(name=self.enterprise_name)
        response = self.client.get(
            reverse('organization_id', args=[enterprise.id])
        )
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, enterprise)

#  TestCase для проверки списка доступных организаций
class EnterpriseListTest(TestCase):
    def setUp(self):
        self.client = APIClient()

        #  Создание обьектов для модели "Районы"
        District.objects.create(name='District 1')
        self.district_1 = District.objects.get(name='District 1').id

        District.objects.create(name='District 2')
        self.district_2 = District.objects.get(name='District 2').id

        #  Создание обьектов для модели "Сеть предприятий"
        EnterpriseNetwork.objects.create(name='Enterprise network 1')
        self.enterprise_network_1 = EnterpriseNetwork.objects.get(name='Enterprise network 1').id

        EnterpriseNetwork.objects.create(name='Enterprise network 2')
        self.enterprise_network_2 = EnterpriseNetwork.objects.get(name='Enterprise network 2').id

        #  Предприятия 1
        self.enterprise_name_1 = 'Enterprise 1'
        Enterprise.objects.create(
            id=1,
            district=[self.district_1],
            name=self.enterprise_name_1,
            description='Description for Enterprise 1',
            affiliation=[self.enterprise_network_1],
        )

        #  Предприятие 2
        self.enterprise_name_2 = 'Enterprise 2'
        Enterprise.objects.create(
            id=2,
            district=[self.district_1],
            name=self.enterprise_name_2,
            description='Description for Enterprise 2',
            affiliation=[self.enterprise_network_2],
        )

        #  Предприятие 3
        self.enterprise_name_3 = 'Enterprise 3'
        Enterprise.objects.create(
            id=3,
            district=[self.district_2],
            name=self.enterprise_name_3,
            description='Description for Enterprise 3',
            affiliation=[self.enterprise_network_1],
        )

        #  Предприятие 4
        self.enterprise_name_4 = 'Enterprise 4'
        Enterprise.objects.create(
            id=4,
            district=[self.district_2],
            name=self.enterprise_name_4,
            description='Description for Enterprise 4',
            affiliation=[self.enterprise_network_2],
        )

    def test_get_companies_list(self):
        enterprises = Enterprise.objects.all().values('name')
        response = self.client.get(
            reverse('organizations'),
        )
        response_list = list()
        for i in json.loads(response.content):
            response_list.append({'name': i['name']})

        self.assertEqual(response.status_code, 200)
        self.assertListEqual(
            response_list,
            list(enterprises)
        )


# coding: UTF-8
import sae
sae.add_vendor_dir('vendor')
import os
import web
from WeixinInterface import WeixinInterface

web.config.debug = True

urls = (
'/weixin','WeixinInterface'
)

app_root = os.path.dirname(__file__)
templates_root = os.path.join(app_root, 'templates')
render = web.template.render(templates_root)

app = web.application(urls, globals()).wsgifunc()

'''
def app(environ, start_response):
    status = '200 OK'
    response_headers = [('Content-type', 'text/plain')]
    start_response(status, response_headers)
    return ['Hello, world!']
'''

application = sae.create_wsgi_app(app)


import os, sys

lib_path = os.path.abspath(os.path.join('..'))
sys.path.append(lib_path)

import unittest
from json2html import *

class TestJson2Html(unittest.TestCase):

    def setUp(self):
        self.test_json = []
        self.path = os.path.dirname(os.path.abspath('.'))

        self.jsonFiles = []
        self.jsonFiles += [each.split('.')[0] for each in os.listdir('.') if each.endswith('.json')]

        for _file in self.jsonFiles:
            fpath = os.path.join(self.path + '/test', _file)
            self.test_json.append({
                'json': open('%s.json' % fpath).read(),
                'output': open('%s.txt' % fpath).read(),
            })

    def tearDown(self):
        pass

    '''
        Exception handling tests
    '''
    def test_empty_json_exception(self, *args, **kwargs):
        _json = ''
        with self.assertRaises(Exception) as context:
            json2html.convert(json = _json)

        self.assertIn('Can\'t convert NULL!', context.exception)

    def test_invalid_json_exception(self, *args, **kwargs):
        _json = {'name'}
        with self.assertRaises(TypeError) as context:
            json2html.convert(json = _json)

        self.assertIn('is not JSON serializable', str(context.exception))

    def test_dict_instead_of_list_exception(self, *args, **kwargs):
        _json = [{}]
        with self.assertRaises(Exception) as context:
            json2html.convert(json = _json)

        self.assertIn('Not a valid JSON list', str(context.exception))

    def test_all(self):
        for i in range(0, len(self.test_json)):
            _json = self.test_json[i]['json']
            output = json2html.convert(json = _json)

            self.assertEqual(str(output), self.test_json[i]['output'])

if __name__ == '__main__':
    unittest.main()


import json
import numpy
import pandas

def read_file_label(fname):
    L = []
    with open(fname, 'r') as fpt:
        l = fpt.readlines()
    for row, d in enumerate(l):
        try:
            a = json.loads(str(d, encoding='utf-8'))
        except TypeError:
            a = json.loads(d)
        L.append( a['klass'] )
    return L
def read_file_decision_function(fname):
    L = []
    with open(fname, 'r') as fpt:
        l = fpt.readlines()
    X = numpy.zeros((len(l),4),float)
    for row, d in enumerate(l):
        try:
            a = json.loads(str(d, encoding='utf-8'))
        except TypeError:
            a = json.loads(d)
        X[row,:] = a['decision_function']
    return X

ftrain = 'data/train.json'
ftest = 'data/test.json'
fpiotrain = 'data/external_data/pioanalisis_train.json'
fpiotest = 'data/external_data/pioanalisis_test.json'
fspatrain = 'data/external_data/spanish_dataset_HA_train.json'
fspatest = 'data/external_data/spanish_dataset_HA_test.json'
ftastrain = 'data/external_data/tass2017_train.json'
ftastest = 'data/external_data/3-32.train.json.predicted.json'

ftas2016train = 'data/external_data/tass2016_train.json'
ftas2016test = 'data/external_data/tass2016_test.json'
ftas2017sctrain = 'data/external_data/tass2017_stemcomp_train.json'
ftas2017sctest = 'data/external_data/tass2017_stemcomp_test.json'

Ytrain = read_file_label(ftrain)
Ytest = read_file_label(ftest)
Xtrain_tas = read_file_decision_function(ftastrain)
Xtrain_pio = read_file_decision_function(fpiotrain)
Xtrain_spa = read_file_decision_function(fspatrain)
Xtrain_tas2016 = read_file_decision_function(ftas2016train)
Xtrain_tas2017sc = read_file_decision_function(ftas2017sctrain)
Xtest_tas = read_file_decision_function(ftastest)
Xtest_pio = read_file_decision_function(fpiotest)
Xtest_spa = read_file_decision_function(fspatest)
Xtest_tas2016 = read_file_decision_function(ftas2016test)
Xtest_tas2017sc = read_file_decision_function(ftas2017sctest)
Xtrain = numpy.concatenate( (Xtrain_tas,Xtrain_pio,Xtrain_spa,Xtrain_tas2016,Xtrain_tas2017sc),axis=1 )
Xtest = numpy.concatenate( (Xtest_tas,Xtest_pio,Xtest_spa,Xtest_tas2016,Xtest_tas2017sc),axis=1 )

df = pandas.DataFrame( Xtrain )
df['Y'] = Ytrain
df.to_csv('data/decision_function/train.csv',header=None,index=False)

df = pandas.DataFrame( Xtest )
df.to_csv('data/decision_function/test_data.csv',header=None,index=False)

df = pandas.DataFrame( Ytest )
df.to_csv('data/decision_function/test_labels.csv',header=None,index=False)


#!/usr/bin/env python

import numpy as np
import matplotlib.pyplot as plt
import rospy
import os
import math
import time
from geometry_msgs.msg import Twist
from sensor_msgs.msg import JointState
from std_msgs.msg import Float64
from std_msgs.msg import Bool
from quadruped_driver.msg import LegState

'''

                 m1 |-------------------| m1
          Leg1      |     F R O N T     |      Leg3              /|\ x
                 m2 |                   | m2                      |
                    |                   |                         |
                    |                   |                  <----- o
                  LEFT                RIGHT                y
                    |                   |
                    |                   |
                 m1 |                   | m1
          Leg2      |      B A C K      |      Leg4
                 m2 |-------------------| m2

'''

    # ..... Geometrical .....
r1              = 20.0               # length of Cranks
r2              = 40.0               # length of Coupler Links
L               = 5.0                # Distance between the pivot points
base_l          = 60.0               # Base length
base_w          = 35.0               # Base width
d               = 0.0                # Offset distance between side edge and foot center
xc1 =  L/2.0; yc1 = 0.0              # Joint-1 coordinates
xc2 = -L/2.0; yc2 = 0.0              # Joint-2 coordinates

    # ..... Trajectory .....
stroke_h        = 36.0               # Robot Height
ht              = 12.0                # Step height

    # ..... Temporal .....
def_period      = 1.0                # default period
dc              = 0.8                # Duty factor
dt              = 0.03               # Extension time
phi1            = +0.5               # Phase diff. b/w Contralateral legs (Leg 1 and Leg 4, Leg 2 and Leg 3)
phi2            = +0.75              # Phase diff. b/w Ipsilateral legs  (Leg 1 and Leg 3, Leg 2 and Leg 4)
step_size       = 0.002
n               = 0
transn_idx      = 0

leg_phase       = np.array([0, 0.5, 0.25, 0.75], dtype=float)

NUM_MOTORS                  = 8
NUM_LEGS                    = 4
MOTOR_AXIS                  = np.ones(8, dtype=int)
MOTOR_OFFSETS               = np.array([0 for i in range(NUM_MOTORS)],dtype=int)
JOINT_LIMITS                = np.array([[0,4095],[0,4095]])

prev_vel        = Twist()
curr_vel        = Twist()
reso_ang        = 0.01
reso_lin        = 0.001
isChanged       = False
isReceived      = 4
motor_angles    = np.zeros(8)
leg_offset      = np.array((leg_phase/step_size),dtype=int)

joint_list = ["leg1_motor_1","leg1_motor_2","leg2_motor_1","leg2_motor_2","leg3_motor_1","leg3_motor_2","leg4_motor_1", "leg4_motor_2"]

class HandlerClass:

    pub_list = []
    leg_state_pub = None
    def __init__(self):
        for joint in joint_list:
            topic = "quadruped/" + joint + "_position_controller/command"
            pub = rospy.Publisher(topic, Float64, queue_size=10)
            self.pub_list.append(pub)
        self.leg_state_pub = rospy.Publisher("/quadruped/leg_contact", LegState, queue_size=10)

motor_handler   = HandlerClass()

def write_motor_angles():
    goal_pos = get_goal_pos()
    execute_goal_pos(goal_pos)

def write_leg_state(idx_t):
    i = (idx_t+leg_offset[:])%n
    lim_max = dc*n/2
    lim_min = n - dc*n/2
    k = (i > lim_min) | (i < lim_max)
    motor_handler.leg_state_pub.publish(k)

def get_goal_pos():
    goal_pos = motor_angles*MOTOR_AXIS + MOTOR_OFFSETS
    return goal_pos

def move_motors():
    goal_pos_steps = get_goal_steps()
    for goal_pos in goal_pos_steps:
        execute_goal_pos(goal_pos)
        time.sleep(0.01)

def get_goal_steps():
    num_steps = 100
    goal_pos_steps  = []

    current_pos = get_current_pose()
    goal_pos = motor_angles*MOTOR_AXIS + MOTOR_OFFSETS
    diff_step = (goal_pos-current_pos)/num_steps

    for i in range(num_steps):
        goal_pos_steps.append(current_pos + (i+1)*diff_step)
    return goal_pos_steps

def get_current_pose():
    joint_state = rospy.wait_for_message('/quadruped/joint_states', JointState, timeout=None)
    pos = np.array(joint_state.position,dtype=float)
    return pos

def calculate_speeds(goal, current, time):
    diff = np.absolute((goal - current), dtype=float)
    speeds = diff/time
    speeds = np.array(speeds,dtype=int)
    return speeds

def execute_goal_pos(goal):
    for i in range(NUM_MOTORS):
        pub = motor_handler.pub_list[i]
        pub.publish(goal[i])

def getParams():
    global n
    global transn_idx

    lin_vel = 100*curr_vel.linear.x
    ang_vel = curr_vel.angular.z

    #params = [stroke, ht, period]
    l_params = np.zeros(3)
    r_params = np.zeros(3)

    vl = lin_vel - base_w*ang_vel/2.0
    vr = lin_vel + base_w*ang_vel/2.0

    if abs(vl) > reso_lin:
        l_params[0] = dc*def_period*vl
        l_params[1] = (2*abs(l_params[0])/8 + 1)
    if abs(vr) > reso_lin:
        r_params[0] = dc*def_period*vr
        r_params[1] = (2*abs(r_params[0])/8 + 1)

    l_params[2] = def_period
    r_params[2] = def_period

    n = int(def_period/step_size)
    transn_idx = (n-leg_offset)%n

    return [l_params, r_params]

def getTrajectory(paramTraj):
    stroke, ht, period = paramTraj

    C_x = np.array([              - (stroke/2) - (dt*stroke)/(dc*period),
                                                    - stroke/(dc*period),
                      (3*stroke)/(dc*pow((2*dt - period + dc*period),2)),
                     (2*stroke)/(dc*pow((2*dt - period + dc*period),3))],
                                                             dtype=float)

    C_y = np.array([                               -stroke_h,
                                                           0,
                            (16*ht)/pow((period*(dc - 1)),2),
                            (32*ht)/pow((period*(dc - 1)),3),
                           (16*ht)/pow((period*(dc - 1)),4)],
                                                 dtype=float)

    Time = np.arange(0,period,step_size)
    X = np.zeros(n); Y = np.zeros(n)

    #.......... X vs Time ..........#
    Phase_x = np.array([((0.5*dc*period+dt)/step_size), ((period-(dt+0.5*dc*period))/step_size)], dtype=int)
    X[0:Phase_x[0]] = -stroke*Time[0:Phase_x[0]]/(dc*period)
    t = Time[Phase_x[0]:Phase_x[1]] - (0.5*dc*period+dt)
    X[Phase_x[0]:Phase_x[1]] = np.dot((np.vstack((np.ones(t.size), t, np.power(t,2), np.power(t,3))).T),C_x)
    X[Phase_x[1]:] = stroke*(period - Time[Phase_x[1]:])/(dc*period)

    #.......... Y vs Time ..........#
    Phase_y = np.array([(0.5*dc*period)/step_size, ((1 - 0.5*dc)*period)/step_size], dtype=int)
    Y[0:Phase_y[0]] = -1*stroke_h*np.ones(Phase_y[0])
    t = Time[Phase_y[0]:Phase_y[1]] - (0.5*dc*period)
    Y[Phase_y[0]:Phase_y[1]] = np.dot((np.vstack((np.ones(t.size), t, np.power(t,2), np.power(t,3), np.power(t,4))).T),C_y)
    Y[Phase_y[1]:] = -1*stroke_h*np.ones(n-Phase_y[1])

    #return np.column_stack((X, Y, Time))
    return [X, Y, Time]

def subplotTraj(x,y,idx,name):
    plt.subplot(idx)
    plt.plot(x, y);
    plt.title(name);

def plotTraj(X_l, Y_l, Time_l, X_r, Y_r, Time_r):
    subplotTraj(Time_l, X_l, 231, 'Left X vs Time')
    subplotTraj(Time_l, Y_l, 232, 'Left Y vs Time')
    subplotTraj(X_l, Y_l, 233, 'Left Trajectory')

    subplotTraj(Time_r, X_r, 234, 'Right X vs Time')
    subplotTraj(Time_r, Y_r, 235, 'Right Y vs Time')
    subplotTraj(X_r, Y_r, 236, 'Right Trajectory')

    plt.show()

def solveIK(x,y):
    a_sq = np.power((x - xc1),2) + np.power((y - yc1),2);
    b_sq = np.power((x - xc2),2) + np.power((y - yc2),2);
    a = np.sqrt(a_sq)
    b = np.sqrt(b_sq)

    o1 = np.arccos((-(r2*r2) + (r1*r1) + b_sq)/(2*r1*b));
    o2 = np.arccos((-(r2*r2) + (r1*r1) + a_sq)/(2*r1*a));
    g1 = np.arccos(-(a_sq - (L*L) - b_sq)/(2*L*b));
    g2 = np.arccos(-(b_sq - (L*L) - a_sq)/(2*L*a));
    Q2 = -np.pi+(g1+o1);
    Q1 = -np.pi+(g2+o2);

    return [Q1, Q2]

def saveData(pos, var):
    np.savetxt(pos, var, delimiter=",")

def cmd_vel_listener():
    global curr_vel
    rospy.init_node('base_controller', anonymous=True)
    curr_vel = rospy.wait_for_message("cmd_vel", Twist, timeout=None)
    rospy.Subscriber("cmd_vel", Twist, callback)

def callback(data):
    global prev_vel
    global curr_vel
    global isChanged
    curr_vel = data
    if abs(curr_vel.linear.x - prev_vel.linear.x) > reso_lin or abs(curr_vel.angular.z - prev_vel.angular.z) > reso_ang:
        prev_vel = curr_vel
        isChanged = True

def solver():
    l_params, r_params = getParams()
    [X_l, Y_l, Time_l] = getTrajectory(l_params)
    [X_r, Y_r, Time_r] = getTrajectory(r_params)
    print(l_params)
    print(r_params)
    #plotTraj(X_l, Y_l, Time_l, X_r, Y_r, Time_r)
    [Q1_l, Q2_l] = solveIK(X_l,Y_l)
    [Q1_r, Q2_r] = solveIK(X_r,Y_r)

    return [Q1_l, Q2_l, Q1_r, Q2_r]

def change_motion_plan(idx_t, motion_plan, new_motion_plan):
    global isReceived
    isReceived = isReceived + 1
    if idx_t == transn_idx[0]: motion_plan[0] = new_motion_plan[0]
    elif idx_t == transn_idx[1]: motion_plan[1] = new_motion_plan[1]
    elif idx_t == transn_idx[2]: motion_plan[2] = new_motion_plan[2]
    elif idx_t == transn_idx[3]: motion_plan[3] = new_motion_plan[3]
    else: isReceived = isReceived - 1

def get_motor_angles(motion_plan, idx_t):
    global motor_angles
    motor_angles[0] = motion_plan[0][(idx_t+leg_offset[0])%n]
    motor_angles[1] = motion_plan[1][(idx_t+leg_offset[0])%n]
    motor_angles[2] = motion_plan[0][(idx_t+leg_offset[1])%n]
    motor_angles[3] = motion_plan[1][(idx_t+leg_offset[1])%n]
    motor_angles[4] = motion_plan[2][(idx_t+leg_offset[2])%n]
    motor_angles[5] = motion_plan[3][(idx_t+leg_offset[2])%n]
    motor_angles[6] = motion_plan[2][(idx_t+leg_offset[3])%n]
    motor_angles[7] = motion_plan[3][(idx_t+leg_offset[3])%n]

if __name__ == '__main__':

    cmd_vel_listener()
    motion_plan = solver()
    idx_t = int(0.625*n)
    get_motor_angles(motion_plan, idx_t)
    print(motor_angles)
    move_motors()
    time.sleep(0.1)
    step_last_time = time.time()

    while not rospy.is_shutdown():

        if isChanged:
            new_motion_plan = solver()
            isChanged = False
            isReceived = 0

        if isReceived != 4:
            change_motion_plan(idx_t, motion_plan, new_motion_plan)

        get_motor_angles(motion_plan, idx_t)
        time.sleep(max(0,step_size-step_last_time))
        write_motor_angles()
        step_last_time = time.time()
        write_leg_state(idx_t)

        idx_t = (idx_t+1)%n


#!/usr/bin/env python
#
# Public Domain 2014-present MongoDB, Inc.
# Public Domain 2008-2014 WiredTiger, Inc.
#
# This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#

import datetime, inspect, os, random, wiredtiger

# These routines help run the various storage sources. They are required to manage
# generation of storage source specific configurations.

# Generate a storage store specific authentication token.
def get_auth_token(storage_source):
    auth_token = None
    if storage_source == 'dir_store':
        # Fake a secret token.
        auth_token = "Secret"
    if storage_source == 's3_store':
        # Auth token is the AWS access key ID and the AWS secret key as semi-colon separated values.
        # We expect the values to have been provided through the environment variables.
        access_key = os.getenv('AWS_ACCESS_KEY_ID')
        secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
        if access_key and secret_key:
            auth_token = access_key + ";" + secret_key
    return auth_token

# Get buckets configured for the storage source

# S3 buckets with their regions
s3_buckets = ['s3testext-us;us-east-2', 's3testext;ap-southeast-2']

# Local buckets do not have a region
local_buckets = ['bucket1', 'bucket2']

# Get name of the bucket at specified index in the list.
def get_bucket_name(storage_source, i):
    if storage_source == 's3_store':
        return s3_buckets[i]
    if storage_source == 'dir_store':
        return local_buckets[i]
    return None

# Set up configuration
def get_conn_config(storage_source):
    if not storage_source.is_tiered_scenario():
            return ''
    if storage_source.ss_name == 'dir_store' and not os.path.exists(storage_source.bucket):
            os.mkdir(storage_source.bucket)
    return \
        'debug_mode=(flush_checkpoint=true),' + \
        'statistics=(all),' + \
        'tiered_storage=(auth_token=%s,' % storage_source.auth_token + \
        'bucket=%s,' % storage_source.bucket + \
        'bucket_prefix=%s,' % storage_source.bucket_prefix + \
        'name=%s,' % storage_source.ss_name

def get_check(storage_source, tc, base, n):
    for i in range(base, n):
        storage_source.assertEqual(tc[str(i)], str(i))
    tc.set_key(str(n))
    storage_source.assertEquals(tc.search(), wiredtiger.WT_NOTFOUND)

# Generate a unique object prefix for the S3 store.
def generate_s3_prefix(test_name = ''):
    # Generates a unique prefix to be used with the object keys, eg:
    # "s3test/python/2022-31-01-16-34-10/623843294--".
    # Objects with the prefex pattern "s3test/*" are deleted after a certain period of time
    # according to the lifecycle rule on the S3 bucket. Should you wish to make any changes to the
    # prefix pattern or lifecycle of the object, please speak to the release manager.
    prefix = 's3test/python/'
    prefix += datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
    # Range upto int32_max, matches that of C++'s std::default_random_engine
    prefix += '/' + str(random.randrange(1, 2147483646)) + '--'

    # If the calling function has not provided a name, extract it from the stack.
    # It is important to generate unique prefixes for different tests in the same class,
    # so that the database namespace do not collide.
    # 0th element on the stack is the current function. 1st element is the calling function.
    if not test_name:
        test_name = inspect.stack()[1][3]
    prefix += test_name + '--'
    return prefix

# Storage sources
tiered_storage_sources = [
    ('dirstore', dict(is_tiered = True,
        is_local_storage = True,
        auth_token = get_auth_token('dir_store'),
        bucket = get_bucket_name('dir_store', 0),
        bucket1 = get_bucket_name('dir_store', 1),
        bucket_prefix = "pfx_",
        bucket_prefix1 = "pfx1_",
        bucket_prefix2 = 'pfx2_',
        num_ops=100,
        ss_name = 'dir_store')),
    ('s3', dict(is_tiered = True,
        is_local_storage = False,
        auth_token = get_auth_token('s3_store'),
        bucket = get_bucket_name('s3_store', 0),
        bucket1 = get_bucket_name('s3_store', 1),
        bucket_prefix = generate_s3_prefix(),
        bucket_prefix1 = generate_s3_prefix(),
        bucket_prefix2 = generate_s3_prefix(),
        num_ops=20,
        ss_name = 's3_store')),
    ('non_tiered', dict(is_tiered = False)),
]

# Sublist to use for the tiered test scenarios as last item on list is not a scenario.
storage_sources = tiered_storage_sources[:2]

# This mixin class provides tiered storage configuration methods.
class TieredConfigMixin:
    # Returns True if the current scenario is tiered.
    def is_tiered_scenario(self):
        return hasattr(self, 'is_tiered') and self.is_tiered

    # Setup connection config.
    def conn_config(self):
        return self.tiered_conn_config()

    # Setup tiered connection config.
    def tiered_conn_config(self):
        # Handle non_tiered storage scenarios.
        if not self.is_tiered_scenario():
            return ''

        # Setup directories structure for local tiered storage.
        if self.is_local_storage:
            bucket_full_path = os.path.join(self.home, self.bucket)
            if not os.path.exists(bucket_full_path):
                os.mkdir(bucket_full_path)

        # Build tiered storage connection string.
        return \
            'debug_mode=(flush_checkpoint=true),' + \
            'tiered_storage=(auth_token=%s,' % self.auth_token + \
            'bucket=%s,' % self.bucket + \
            'bucket_prefix=%s,' % self.bucket_prefix + \
            'name=%s),' % self.ss_name

    # Load the storage sources extension.
    def conn_extensions(self, extlist):
        return self.tiered_conn_extensions(extlist)

    # Load tiered storage source extension.
    def tiered_conn_extensions(self, extlist):
        # Handle non_tiered storage scenarios.
        if not self.is_tiered_scenario():
            return ''

        config = ''
        # S3 store is built as an optional loadable extension, not all test environments build S3.
        if not self.is_local_storage:
            #config = '=(config=\"(verbose=1)\")'
            extlist.skip_if_missing = True
        #if self.is_local_storage:
            #config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")'
        # Windows doesn't support dynamically loaded extension libraries.
        if os.name == 'nt':
            extlist.skip_if_missing = True
        extlist.extension('storage_sources', self.ss_name + config)

    # Wrapper around session.alter call
    def alter(self, uri, alter_param):
        # Tiered storage does not fully support alter operation. FIXME WT-9027.
        try:
            self.session.alter(uri, alter_param)
        except BaseException as err:
            if self.is_tiered_scenario() and str(err) == 'Operation not supported':
                self.skipTest('Tiered storage does not fully support alter operation.')
            else:
                raise


# Copyright 2018 The Cirq Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Abstract base class for things sampling quantum circuits."""

from typing import List, Optional, TYPE_CHECKING, Union
import abc

import pandas as pd

from cirq import study

if TYPE_CHECKING:
    import cirq

class Sampler(metaclass=abc.ABCMeta):
    """Something capable of sampling quantum circuits. Simulator or hardware."""

    def run(
            self,
            program: 'cirq.Circuit',
            param_resolver: 'cirq.ParamResolverOrSimilarType' = None,
            repetitions: int = 1,
    ) -> 'cirq.TrialResult':
        """Samples from the given Circuit.

        By default, the `run_async` method invokes this method on another
        thread. So this method is supposed to be thread safe.

        Args:
            program: The circuit to sample from.
            param_resolver: Parameters to run with the program.
            repetitions: The number of times to sample.

        Returns:
            TrialResult for a run.
        """
        return self.run_sweep(program, study.ParamResolver(param_resolver),
                              repetitions)[0]

    def sample(
            self,
            program: 'cirq.Circuit',
            *,
            repetitions: int = 1,
            params: 'cirq.Sweepable' = None,
    ) -> 'pd.DataFrame':
        """Samples the given Circuit, producing a pandas data frame.

        Args:
            program: The circuit to sample from.
            repetitions: The number of times to sample the program, for each
                parameter mapping.
            params: Maps symbols to one or more values. This argument can be
                a dictionary, a list of dictionaries, a `cirq.Sweep`, a list of
                `cirq.Sweep`, etc. The program will be sampled `repetition`
                times for each mapping. Defaults to a single empty mapping.

        Returns:
            A `pandas.DataFrame` with a row for each sample, and a column for
            each measurement key as well as a column for each symbolic
            parameter.  Measurement results are stored as a big endian integer
            representation with one bit for each measured qubit in the key.
            See `cirq.big_endian_int_to_bits` and similar functions for how
            to convert this integer into bits.
            There is an also index column containing the repetition number,
            for each parameter assignment.

        Examples:
            >>> a, b, c = cirq.LineQubit.range(3)
            >>> sampler = cirq.Simulator()
            >>> circuit = cirq.Circuit(cirq.X(a),
            ...                        cirq.measure(a, key='out'))
            >>> print(sampler.sample(circuit, repetitions=4))
               out
            0    1
            1    1
            2    1
            3    1

            >>> circuit = cirq.Circuit(cirq.X(a),
            ...                        cirq.CNOT(a, b),
            ...                        cirq.measure(a, b, c, key='out'))
            >>> print(sampler.sample(circuit, repetitions=4))
               out
            0    6
            1    6
            2    6
            3    6

            >>> circuit = cirq.Circuit(cirq.X(a)**sympy.Symbol('t'),
            ...                        cirq.measure(a, key='out'))
            >>> print(sampler.sample(
            ...     circuit,
            ...     repetitions=3,
            ...     params=[{'t': 0}, {'t': 1}]))
               t  out
            0  0    0
            1  0    0
            2  0    0
            0  1    1
            1  1    1
            2  1    1
        """

        sweeps_list = study.to_sweeps(params)
        keys = sorted(sweeps_list[0].keys) if sweeps_list else []
        for sweep in sweeps_list:
            if sweep and set(sweep.keys) != set(keys):
                raise ValueError(
                    'Inconsistent sweep parameters. '
                    f'One sweep had {repr(keys)} '
                    f'while another had {repr(sorted(sweep.keys))}.')

        results = []
        for sweep in sweeps_list:
            sweep_results = self.run_sweep(program,
                                           params=sweep,
                                           repetitions=repetitions)
            for resolver, result in zip(sweep, sweep_results):
                param_values_once = [resolver.value_of(key) for key in keys]
                param_table = pd.DataFrame(data=[param_values_once] *
                                           repetitions,
                                           columns=keys)
                results.append(pd.concat([param_table, result.data], axis=1))

        return pd.concat(results)

    @abc.abstractmethod
    def run_sweep(
            self,
            program: 'cirq.Circuit',
            params: 'cirq.Sweepable',
            repetitions: int = 1,
    ) -> List['cirq.TrialResult']:
        """Samples from the given Circuit.

        In contrast to run, this allows for sweeping over different parameter
        values.

        Args:
            program: The circuit to sample from.
            params: Parameters to run with the program.
            repetitions: The number of times to sample.

        Returns:
            TrialResult list for this run; one for each possible parameter
            resolver.
        """

    async def run_async(self, program: 'cirq.Circuit', *,
                        repetitions: int) -> 'cirq.TrialResult':
        """Asynchronously samples from the given Circuit.

        By default, this method invokes `run` synchronously and simply exposes
        its result is an awaitable. Child classes that are capable of true
        asynchronous sampling should override it to use other strategies.

        Args:
            program: The circuit to sample from.
            repetitions: The number of times to sample.

        Returns:
            An awaitable TrialResult.
        """
        return self.run(program, repetitions=repetitions)

    async def run_sweep_async(
            self,
            program: 'cirq.Circuit',
            params: 'cirq.Sweepable',
            repetitions: int = 1,
    ) -> List['cirq.TrialResult']:
        """Asynchronously sweeps and samples from the given Circuit.

        By default, this method invokes `run_sweep` synchronously and simply
        exposes its result is an awaitable. Child classes that are capable of
        true asynchronous sampling should override it to use other strategies.

        Args:
            program: The circuit to sample from.
            params: One or more mappings from parameter keys to parameter values
                to use. For each parameter assignment, `repetitions` samples
                will be taken.
            repetitions: The number of times to sample.

        Returns:
            An awaitable TrialResult.
        """
        return self.run_sweep(program, params=params, repetitions=repetitions)

    def run_batch(
            self,
            programs: List['cirq.Circuit'],
            params_list: Optional[List['cirq.Sweepable']] = None,
            repetitions: Union[int, List[int]] = 1,
    ) -> List[List['cirq.TrialResult']]:
        """Runs the supplied circuits.

        Each circuit provided in `programs` will pair with the optional
        associated parameter sweep provided in the `params_list`, and be run
        with the associated repetitions provided in `repetitions` (if
        `repetitions` is an integer, then all runs will have that number of
        repetitions). If `params_list` is specified, then the number of
        circuits is required to match the number of sweeps. Similarly, when
        `repetitions` is a list, the number of circuits is required to match
        the length of this list.

        By default, this method simply invokes `run_sweep` sequentially for
        each (circuit, parameter sweep, repetitions) tuple. Child classes that
        are capable of sampling batches more efficiently should override it to
        use other strategies. Note that child classes may have certain
        requirements that must be met in order for a speedup to be possible,
        such as a constant number of repetitions being used for all circuits.
        Refer to the documentation of the child class for any such requirements.

        Args:
            programs: The circuits to execute as a batch.
            params_list: Parameter sweeps to use with the circuits. The number
                of sweeps should match the number of circuits and will be
                paired in order with the circuits.
            repetitions: Number of circuit repetitions to run. Can be specified
                as a single value to use for all runs, or as a list of values,
                one for each circuit.

        Returns:
            A list of lists of TrialResults. The outer list corresponds to
            the circuits, while each inner list contains the TrialResults
            for the corresponding circuit, in the order imposed by the
            associated parameter sweep.
        """
        if params_list is None:
            params_list = [None] * len(programs)
        if len(programs) != len(params_list):
            raise ValueError('len(programs) and len(params_list) must match. '
                             f'Got {len(programs)} and {len(params_list)}.')
        if isinstance(repetitions, int):
            repetitions = [repetitions] * len(programs)
        if len(programs) != len(repetitions):
            raise ValueError('len(programs) and len(repetitions) must match. '
                             f'Got {len(programs)} and {len(repetitions)}.')
        return [
            self.run_sweep(circuit, params=params, repetitions=repetitions)
            for circuit, params, repetitions in zip(programs, params_list,
                                                    repetitions)
        ]