import numpy as np

# Define x and y grid size. Must match xnum and ynum in "Train.py"
xnum = 18
ynum = 55

#  Function calculates class of each ADP based on the loc data
# Inputs: raw validation data (ADP, location), and size of (x,y) grid
# t - handles name of location dataset. Can be removed if name of location column is known
# Outputs: (ADP, class) pairs and new location based on classes (similar to get_data in tarin.py)
def get_valid_data(data,xnum,ynum,t):
  # Calculate total number of classes
  num_classes = xnum*ynum
  adp = data['ADP']
  print('adp shape ',adp.shape)
  # Get ADP and (x,y) location from dataset
  M = np.reshape(adp,(-1,32,32))
  if t ==0 :
    loc = data['Location']
  elif t ==1:
    loc = data['Loc']
  x = loc[:,0]
  y = loc[:,1]
  # Create placeholders for class (grid) and new (X,Y) coordiantes which are at the center of each grid.
  # Example: If the user is in the mth segment in x-direaction and nth segment in y-direction, the new coordinates are (m,n) and the sample belongs to class c = m*n.
  cnew = np.zeros((len(loc),num_classes))
  c = np.zeros(len(loc))
  xnew = np.zeros(len(x))
  ynew = np.zeros(len(y))

   # Calculate step size based on the grid
  xmax = max(x)
  xmin = min(x)
  xstep = (xmax-xmin)/xnum
  ymax = max(y)
  ymin = min(y)
  ystep = (ymax-ymin)/ynum

  # Convert x coordinate to grid segment in x
  for i in range(len(x)):
      for j in range(1,xnum+1):
          low = xmin + (j-1)*xstep
          high = xmin + j * xstep
          if(x[i]>=low and x[i]<high):
              xnew[i] = j
          if(x[i] == xmax):
              xnew[i] = xnum
      if (xnew[i] == 0):
          print(x[i])
          print(xmax)
          print(xmin)
          sys.exit()

 # Convert y coordinate to grid segment in y
  for i in range(len(y)):
      for j in range(1,ynum+1):
          low = ymin + (j-1)*ystep
          high = ymin + (j*ystep)
          if(y[i]>=low and y[i]<high):
              ynew[i] = j
          if(y[i] == ymax):
              ynew[i] = ynum
      if (ynew[i] == 0):
          print(y[i])
          print(ymax)
          print(ymin)
          sys.exit()

   # Assign classes to dataset based on grid starting with grid (1,1) assigned to class c=1
   # and grid (m,n) assigned to class c = m*n.
  for i in range(len(loc)):
      c[i] = (xnew[i]-1)+xnum*(ynew[i]-1)
  c = np.reshape(c,(len(c),1))

  # M - ADP
  # c - class
  #  new location (x,y) based on grid
  return loc,c, M

# For any grid, collect and group all ADP samples from the training set that belong to that grid
# This means that by calling this function for for each grid there will be a collection of ADPs that belong to that grid.
# This collection will later be used for KNN alogirthm to search within.
# Input:
# subclass - that class for which you want to collect all the ADPs. If you want all the ADPs in the training data beloning to class m, then subclass = m.
# M - All ADPs in the training data
# c - total number of class
# (x,y)- loc from training data for all ADPs
# Outputs:
# Mnew - All ADPs beloning to the selected subclass m
# c1 -  class m
# (xlocnew,ylocnew) - new (x,y) location coordinates based on the grid
def get_sub(subclass,M,c,x,y):
  xnum = 18
  ynum = 55

  n = 0
  for i in range(len(c)):
    if (c[i]==subclass):
      n+=1
  M  = np.reshape(M,[-1,64,64])
  Mnew = np.zeros([n,64,64])
  xlocnew = np.zeros(n)
  ylocnew = np.zeros(n)

  j=0

  # Search for all (ADP, loc) beloning to the subclass
  for i in range((len(c))):
    if (c[i]==subclass):
      Mnew[j,:,:]=M[i,:,:]
      xlocnew[j] = x[i]
      ylocnew[j] = y[i]
      j+=1

  c1 = np.zeros(n)
  xnew = np.zeros(n)
  ynew = np.zeros(n)
  xmax = max(xlocnew)
  xmin = min(xlocnew)
  xstep = (xmax-xmin)/xnum
  ymax = max(ylocnew)
  ymin = min(ylocnew)
  ystep = (ymax-ymin)/ynum

  # Compute new x and y location based on the grid
  for i in range(n):
      for j in range(1,xnum+1):
          low = xmin + (j-1)*xstep
          high = xmin + j * xstep
          if(xlocnew[i]>=low and x[i]<high):
              xnew[i] = j
          if(xlocnew[i] == xmax):
              xnew[i] =xnum

  for i in range(n):
      for j in range(1,ynum+1):
          low = ymin + (j-1)*ystep
          high = ymin + (j*ystep)
          if(ylocnew[i]>=low and ylocnew[i]<high):
              ynew[i] = j
          if(y[i] == ymax):
              ynew[i] = ynum

  #Compute class based on grid
  for i in range(len(c1)):
      c1[i] = (xnew[i]-1)+xnum*(ynew[i]-1)

  c1 = np.reshape(c1,(n,1))

  # Mnew -ADP
  # c1- class
  #xlocnew and ylocnew are x and y coordinates
  return Mnew, c1,xlocnew,ylocnew

# Loads Train Dataset
data_path1 ='Data/TrainDCNNI1.npz'
data1 = np.load(data_path1)

# Takes the dataset and grid size as input and outputs ADP, class, and location (x,y)
loc_t, c_t, ADP_t= get_valid_data(data1,xnum,ynum,1)
num_classes = xnum*ynum
x_loc = loc_t[:,0]
y_loc = loc_t[:,1]

for sub in range(num_classes):
   # For every class on the grid it creates a dataset of (ADP, class, location)
   train_ADP, train_c, xtrain, ytrain = get_sub(sub,ADP_t,c_t,x_loc,y_loc)
   np.save('ADP/ADP'+str(sub),train_ADP)
   np.save('ADP/class'+str(sub),train_c)
   np.save('ADP/xtrain'+str(sub),xtrain)
   np.save('ADP/ytrain'+str(sub),ytrain)


from openerp import models, fields

class Company(models.Model):
    _inherit = 'res.company'

    outgoing_wave_behavior_on_confirm = fields.Selection(
        [
            (0, 'Close pickings in wave with creation of backorders '
                'for incomplete pickings'),
            (1, 'Close pickings in wave without creating backorders'),
            (2, 'Move wave to "On Hold" if not all pickings are confirmed')
        ],
        string='Behavior on Confirm', default=0)


import os

os.system('cls')

colors = {
    "Black": "\u001b[30m",
    "Red": "\u001b[31m",
    "Green": "\u001b[32m",
    "Yellow": "\u001b[33m",
    "Blue": "\u001b[34m",
    "Magenta": "\u001b[35m",
    "Cyan": "\u001b[36m",
    "White": "\u001b[37m",
    "Bright Black": "\u001b[30;1m",
    "Bright Red": "\u001b[31;1m",
    "Bright Green": "\u001b[32;1m",
    "Bright Yellow": "\u001b[33;1m",
    "Bright Blue": "\u001b[34;1m",
    "Bright Magenta": "\u001b[35;1m",
    "Bright Cyan": "\u001b[36;1m",
    "Bright White": "\u001b[37;1m",
    "Background Black": "\u001b[40m",
    "Background Red": "\u001b[41m",
    'Background Green': "\u001b[42m",
    "Background Yellow": "\u001b[43m",
    "Background Blue": "\u001b[44m",
    "Background Magenta": " \u001b[45m",
    "Background Cyan": "\u001b[46m",
    "Background White": "\u001b[47m",
    "Background Bright Black": "\u001b[40;1m",
    "Background Bright Red": "\u001b[41;1m",
    "Background Bright Green": "\u001b[42;1m",
    "Background Bright Yellow": "\u001b[43;1m",
    "Background Bright Blue": " \u001b[44;1m",
    "Background Bright Magenta": "\u001b[45;1m",
    "Background Bright Cyan": "\u001b[46;1m",
    "Background Bright White": "\u001b[47;1m",
    "Reset": "\u001b[0m"
}

def colorText(text):
    for c in colors:
        text = text.replace("[" + c + "]", colors[c])
    return text

# fas = " [Red]Fake [Blue]A [Green]Smile"
fas = open("colourFulText.txt", "r")
ascii = "".join(fas.readlines())
coloredAns = colorText(ascii)
print(coloredAns)


import responses

from tests.util import answers
from tests.rest import client

from selvpcclient.resources.customization import CustomizationManager

@responses.activate
def test_customization_raw_return():
    responses.add(
        responses.GET,
        "http://api/v2/theme",
        json=answers.CUSTOMIZATION_SHOW
    )
    manager = CustomizationManager(client)
    result = manager.show(return_raw=True)
    assert result == answers.CUSTOMIZATION_SHOW['theme']


"""
  @author Victor I. Afolabi
  A.I. engineer/researcher & Software engineer
  javafolabi@gmail.com

  Created on 04 January, 2018 @ 7:28 PM.

  Copyright © 2018. Victor. All rights reserved.
"""

from flask import render_template, request
# helper module for image classification
# import models.projects.image_classification as img_class
# helper module for image search
import models.projects.image_search as img_search
from views import app, back

@app.route('/projects/')
@back.anchor
def projects():
    """Projects page.

    Decorators:
        back
        app

    Returns:
        [template] -- [renders page template]
    """

    return render_template('projects/index.html')

@app.route('/projects/image-classification/', methods=['GET', 'POST'])
@back.anchor
def image_classification():
    """Image classification page.

    Decorators:
        back
        app

    Returns:
        [type] -- [description]
    """
    return render_template('projects/image-classification.html')

@app.route('/projects/generative-models/')
@back.anchor
def generative_models():
    """Generative models page.

    Decorators:
        back
        app

    Returns:
        [template] -- [renders page template]
    """

    return render_template('projects/generative-models.html')

@app.route('/projects/image-search/', methods=['GET', 'POST'])
@back.anchor
def image_search():
    """Image search page.

    Decorators:
        back
        app

    Returns:
        [template] -- [renders page template]
    """

    if request.method == 'POST':
        try:
            data = img_search.process(request)
            return render_template('projects/image-search.html', data=data)
        except Exception as e:
            print(f'ERROR: {e}')
            return render_template('projects/image-search.html', error=e)
    # !- request.method == 'GET'
    return render_template('projects/image-search.html')

@app.route('/projects/ai-articles/')
@back.anchor
def ai_articles():
    """A.I. Articles page.

    Decorators:
        back
        app

    Returns:
        [template] -- [renders page template]
    """

    return render_template('projects/ai-articles.html')

@app.route('/projects/auto-encoding/')
@back.anchor
def auto_encoding():
    """Auto encoding page.

    Decorators:
        back
        app

    Returns:
        [template] -- [renders page template]
    """

    return render_template('projects/auto-encoding.html')

@app.route('/projects/reinforcement-learning/')
@back.anchor
def reinforcement_learning():
    """Reinforcement learning page.

    Decorators:
        back
        app

    Returns:
        [template] -- [renders page template]
    """

    return render_template('projects/reinforcement-learning.html')


from PIL import Image

img = Image.open(r'C:\Users\nscop\Documents\Projects\StatGalaxy\Logo_Rough_Draft.png')
img = img.convert("RGBA")
datas = img.getdata()
threshold = 75

newData = []
for item in datas:
    if item[0] >= threshold and item[1] >= threshold and item[2] >= threshold:
        newData.append((255, 255, 255, 0))
    else:
        newData.append(item)

img.putdata(newData)
img.save(r'C:\Users\nscop\Documents\Projects\StatGalaxy\Logo_Rough_Draft_Transparent.png', "PNG")


# -*- coding: utf-8 -*-
# @Author  : BigBugBoy
# @Software: PyCharm

import requests
from bs4 import BeautifulSoup
import time
from urllib.request import urlopen

# import urllib.request
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
# response = urllib.request.urlopen('https://www.python.org')

'''第一次自己用bs4和request写爬虫'''

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit 537.36 (KHTML, like Gecko)Chrome'}
# headers = {
#     "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0"}
session = requests.session()

# 返回电影相关信息
def movie_info(page):
    print('电影的主链接：%s' % page)
    h = session.get(page, headers=headers)
    soup = BeautifulSoup(h.text, "html.parser")

    # h = urlopen(page)
    # soup = BeautifulSoup(h, 'html.parser')

    # 如果页面不存在，做异常处理 。   **********失败********
    # print(soup)
    # test = soup.find('title').get_text()
    # if test == "页面不存在":
    #     return "movie_athor", " movie_actor", "movie_year", "movie_class"

    result = soup.find('div', {'id': "info"})
    movie_athor_list = []
    movie_actor_list = []
    movie_year_list = []
    movie_class_list = []

    movie_athor = result.span.findAll('a', {'rel': "v:directedBy"})
    for i in range(len(movie_athor)):
        movie_athor_list.append(movie_athor[i].get_text())

    movie_actor = result.findAll('a', {'rel': 'v:starring'})
    for i in range(len(movie_actor)):
        movie_actor_list.append(movie_actor[i].get_text())

    movie_year = result.findAll('span', {'property': 'v:initialReleaseDate'})
    for i in range(len(movie_year)):
        movie_year_list.append(movie_year[i].get_text())

    movie_class = result.findAll('span', {'property': 'v:genre'})
    for i in range(len(movie_class)):
        movie_class_list.append(movie_class[i].get_text())

    return movie_athor_list, movie_actor_list, movie_year_list, movie_class_list

# 主函数
def test_BeautifulSoup(url):
    html = session.get(url, headers=headers)
    # 利用bs4解析网页源代码
    bsOBj = BeautifulSoup(html.text, 'html.parser')

    # 检查编码是否一致
    # print(bsOBj.original_encoding)

    # 找到页面显示电影的模块
    result = bsOBj.find('ol', attrs={'class': "grid_view"})
    # 验证页面源代码
    # print(result)
    # 分别定义存储电影信息的列表
    # 名称
    movie_name_list = []
    # 电影海报
    movie_pic_list = []
    # 标语
    movie_inq_list = []
    # 电影链接
    movie_href_list = []
    # 排名（序号）
    movie_em_list = []
    # 评分
    movie_star_list = []
    # 导演
    athor_list = []
    # 主要演员
    actor_list = []
    # 出品年和国家
    year_list = []
    # 类别
    class_list = []

    # 对电影模块树列进行循环匹配
    for movie_list in result.findAll('li'):
        # 找到电影的名字，加入列表中
        movie_name = movie_list.find(
            'div', {'class': 'hd'}).find('span').get_text()
        movie_name_list.append(movie_name)

        # 找到电影海报，加入列表
        movie_pic = movie_list.find('div', {'class': 'pic'}).find('img')['src']
        movie_pic_list.append(movie_pic)

        # 电影标语
        movie_inq = movie_list.find(
            'p', {'class': 'quote'}).find('span').get_text()
        movie_inq_list.append(movie_inq)

        # 电影链接s
        movie_href = movie_list.find('div', {'class': 'hd'}).find('a')['href']
        movie_href_list.append(movie_href)
        # print(movie_href)

        # 电影排名，加入列表中
        movie_em = movie_list.find(
            'div', {'class': 'pic'}).find('em').get_text()
        movie_em_list.append(movie_em)

        # 电影评分
        movie_star = movie_list.find(
            'span', {'class': 'rating_num'}).get_text()
        movie_star_list.append(movie_star)

        # 保存当前页爬到的图片
        print('当前正在爬取的电影是：%s' % movie_name)
        download_pic(movie_pic, movie_name)
        print('%s的海报图片保存成功' % movie_name)
        print('*****************************************')

        if movie_em in ('20', '27', '122', '123'):
            athor_list.append('movie_athor')
            actor_list.append('movie_actor')
            year_list.append('movie_year')
            class_list.append('movie_class')
        else:
            # 电影制作信息
            movie_athor, movie_actor, movie_year, movie_class = movie_info(
                movie_href)
            athor_list.append(movie_athor)
            actor_list.append(movie_actor)
            year_list.append(movie_year)
            class_list.append(movie_class)

        time.sleep(3)

        '''
        正则表达式，抓取电影制作的相关信息，格式不匹配，一直出现：
        "TypeError: expected string or bytes-like object"的错误异常
        '''
        # 电影制作信息，返回的是一个列表
        # rel = movie_list.find('div', {'class': 'bd'}).find('p').get_text().split()
        # print(rel)

        # res = movie_list.find('div', {'class': 'bd'})
        # print(res)
        # pattern = re.compile(u'主演:(.*?)', re.S)
        # print(re.match(pattern, res))

        # movie = re.match(re.compile('导演:(.*?)'), rel)
        # print(movie)
        # # 制定匹配规则，re.s表示多行匹配
        # pattern = re.compile(u'导演:(.*?)   '       # 导演
        #                      + u'主演:(.*?)
'                   # 主演
        #                      + u'(.*?) / '               # 年份
        #                      + u'(.*?) / '               # 国家
        #                      + u'(.*?)
', re.S)                  # 类别
        # movies = re.findall(pattern, rel)
        # print(movies)
        # for movie in movies:
        #     movie_athor = movie[0].lstrip()
        #     movie_actor = movie[1]
        #     movie_year = movie[2].rstrip()
        #     movie_contry = movie[3]
        #     movie_class = movie[4]
        #     print(movie_athor)
        #     print(movie_actor)
        #     print(movie_year)
        #     print(movie_contry)
        #     print(movie_class)

    '''检查是否查找正确'''
    for i in range(250):
        print('*' * 100)
        print("电影名称：%s" % movie_name_list[i])
        print("电影海报链接：%s" % movie_pic_list[i])
        print("电影豆瓣链接：%s" % movie_href_list[i])
        print("电影排名：%s" % movie_em_list[i])
        print("电影评分：%s" % movie_star_list[i])
        print("电影标语：%s" % movie_inq_list[i])
        print("导演：：%s" % athor_list[i])
        print("电影演员：%s" % actor_list[i])
        print("上映日期和地点：%s" % year_list[i])
        print("电影类别：%s" % class_list[i])
        print('*' * 100)

    # 找到属性class为'title'的标签为span的内容
    # result = bsOBj.findAll('span', {'class' : 'title'})
    # print(result[0])

    # 用css选择器查找(失败)
    # result = bsOBj.select('body>div>div>div>div>ol')
    # print(result)

def download_pic(url, pic_name):  # 下载函数
    name = '/Users/wangwenjun/MyProject/Crawler/Test/' + pic_name + ".jpg"
    print(url)
    print(pic_name)
    if(url is None):  # 地址若为None则跳过
        pass
    result = urlopen(url)  # 打开链接
    # print result.getcode()
    if(result.getcode() != 200):  # 如果链接不正常，则跳过这个链接
        pass
    else:
        data = result.read()  # 否则开始下载到本地
        with open(name, "wb") as code:
            code.write(data)
            code.close()

# 迭代遍历每一页
def page():
    for i in range(0, 10):
        url = 'https://movie.douban.com/top250?start='
        url = url + ('%d' % (i * 25))

        print('\n\n'+'*' * 100)
        print('****************************************爬到的第%d页的数据如下：****************************************' % (i + 1))
        print('*' * 100)

        # 测试单个电影网页不存在的处理
        # url = 'https://movie.douban.com/subject/5912992/'
        # print(url)
        # print(movie_info(url))

        # 正常从主页开始爬
        # print(url)
        test_BeautifulSoup(url)

# 计算爬虫运行时间
def timeup(func):
    start = time.time
    func()
    end = time.time
    timeuse = end - start
    print('\n[%s()]解析一共使用了%d秒时间。\n' % (func.__name__, timeuse))
    return timeuse

def testUrlOpen():
    url = 'https://img3.doubanio.com/view/photo/s_ratio_poster/public/p462657443.webp'
    name = '/Users/wangwenjun/MyProject/Crawler/Test/' + '测试' + ".jpg"
    if(url is None):  # 地址若为None则跳过
        pass
    result = urlopen(url)  # 打开链接
    # print result.getcode()
    if(result.getcode() != 200):  # 如果链接不正常，则跳过这个链接
        pass
    else:
        data = result.read()  # 否则开始下载到本地
        with open(name, "wb") as code:
            code.write(data)
            code.close()

if __name__ == '__main__':
    timeup(page)
    # testUrlOpen()


import os

class ReadTeFile:

    def __init__(self,filename):

        #Parse the input file and store as dictionary, key is line number and value is file contents
        self.dictfile ={}

        self.parsefile(filename)

    def parsefile(self,filename):

        fn  = open(os.path.abspath(filename), "r")
        trlines = fn.readlines()
        fn.close()
        #Key is index, value is 2 element list, first element is label, second element is dictinary with attribute, value pairs
        m = 0
        for line in trlines:
            temp = line.rstrip().split(" ")
            vallist = []
            atval = {}
            for x in range(0,len(temp)):
                if x == 0:
                    temp0int = int(temp[0])
                    vallist.append(temp0int)

                else:
                    attrsplit = temp[x].split(":")
                    attrsplit0int = int(attrsplit[0])
                    attrsplit1int = int(attrsplit[1])
                    atval[attrsplit0int] = attrsplit1int
            vallist.append(atval)
            self.dictfile[m] = vallist
            m = m + 1

    def printtestfile(self):
        for key, val in self.dictfile.items():
            print(key, val)


"""Katie Mishra | July 2019 | katiemishra@gmail.com
Synthesizes speech from the input string of text or ssml.
Exports in wav format, adding one second of silence on each side of clipself.
Requires Google API credential in bash profile. """
from google.cloud import texttospeech
from os import path
from pydub import AudioSegment
import os
from subprocess import call

# Instantiates a client
client = texttospeech.TextToSpeechClient()

#words to generate samples for
words = ["odd","at","hut","ought","cow","hide","be","cheese","thee","Ed","hurt","ate","fee","green","he","it","eat","gee","key","lee","me","knee","ping","oat",
  "toy","pee","read","sea","she","tea","theta","hood","two","vee","we","yield","zee","seizure"]

# create 1 sec of silence audio segment
one_sec_segment = AudioSegment.silent(duration=1000)  #duration in milliseconds

# codes for voices to generate
voice_names = ['en-US-Standard-B','en-US-Standard-C','en-US-Standard-D','en-US-Standard-E','en-US-Wavenet-A','en-US-Wavenet-B','en-US-Wavenet-C','en-US-Wavenet-D','en-US-Wavenet-E','en-US-Wavenet-F']

call("cd ./Samples")

for word in words:

    #create folder and navigate to correct directory
    call("mkdir " + word)
    call("cd " + word)

    # Set the text input to be synthesized
    synthesis_input = texttospeech.types.SynthesisInput(text=word)

    for voice_name in voice_names:
        # Build the voice request, select the language code ("en-US") and the ssml
        # voice gender ("neutral")
        voice = texttospeech.types.VoiceSelectionParams(
                language_code='en-US',
                name=voice_name)

        # Select the type of audio file you want returned
        audio_config = texttospeech.types.AudioConfig(
            audio_encoding=texttospeech.enums.AudioEncoding.MP3)

        # Perform the text-to-speech request on the text input with the selected
        # voice parameters and audio file type
        response = client.synthesize_speech(synthesis_input, voice, audio_config)

        #specify file name based on voice used
        file_type = ".mp3"
        file_name = voice_name + file_type

        # The response's audio_content is binary.
        with open(file_name, 'wb') as out:
            # Write the response to the output file.
            out.write(response.audio_content)
            print('Audio content written to file ' + file_name)

        # file naming
        src = file_name
        dst = voice_name + ".wav"

        # convert mp3 to wav
        sound = AudioSegment.from_mp3(src)
        sound.export(dst, format="wav")
        print('Audio content written to file ' + dst)
        os.remove(src)
        print('Deleted' + src)

        #add silence at beginning and end of wav file (necessary for sphinx analysis)
        #read wav file to an audio segment
        initial_audio = AudioSegment.from_wav(dst)
        #Add above two audio segments
        final_audio = one_sec_segment + initial_audio + one_sec_segment
        #Either save modified audio
        final_audio.export(dst, format="wav")

        call("cd ..")


import os, subprocess
from boto3.session import Session

def connect_to_aws():
    global session

    access_key = input("Copy and paste the compromised AWS access key ID here: ")
    secret_key = input("Copy and paste the compromised AWS secret access key here: ")
    region = input("Enter a region here: ")

    while True:
        try:
            aws_regions = [
                'ap-east-1', 'ap-northeast-1', 'ap-northeast-2', 'ap-south-1', 'ap-southeast-1', 'ap-southeast-2', 'ca-central-1',
                'cn-north-1', 'cn-northwest-1', 'eu-central-1', 'eu-north-1', 'eu-west-1', 'eu-west-2', 'eu-west-3',
                'sa-east-1', 'us-east-1', 'us-east-2', 'us-gov-east-1', 'us-gov-west-1', 'us-west-1', 'us-west-2'
            ]
            region in aws_regions
            break
        except ValueError:
            print("Enter a valid AWS region.")

    session = Session(
        aws_access_key_id='{}'.format(access_key),
        aws_secret_access_key='{}'.format(secret_key),
        region_name='{}'.format(region)
    )

def select_s3_bucket():
    global s3_client, selected_bucket

    s3_client = session.client('s3')
    s3_buckets = s3_client.list_buckets()['Buckets']

    for i in range(len(s3_buckets)):
        print('{0}. {1}'.format(i + 1, s3_buckets[i]['Name']))

    bucket_num = input('Enter the list number of the S3 bucket to download from: ')

    while True:
        try:
            bucket_num = int(bucket_num)
            break
        except ValueError:
            print("This is not a number.")
        try:
            bucket_num <= len(s3_buckets)
            break
        except ValueError:
            print("The number must be less than or equal to the number of S3 buckets.")

    selected_bucket = s3_buckets[bucket_num - 1]['Name']

def download_s3_bucket():
    global bucket_folder

    me = os.getlogin()
    bucket_folder = 'C:\\Users\\{0}\\Downloads\\{1}'.format(me, selected_bucket)

    os.mkdir(bucket_folder, 0o777)

    s3_objects = s3_client.list_objects(Bucket = '{}'.format(selected_bucket))['Contents']

    # Loop through objects in the selected S3 bucket.
    for i in range(len(s3_objects)):
        key_name = s3_objects[i]['Key']

        # Skip past the object if it is a directory.
        if key_name[-1] == '/':
            continue

        nested_dir = '\\'.join(key_name.split('/')[:-1])

        # If the local nested directory already exists, don't remake it.
        if bool(os.path.isdir('{0}\\{1}'.format(bucket_folder, nested_dir))):
            pass
        else:
            os.makedirs('{0}\\{1}'.format(bucket_folder, nested_dir), 0o777)

        s3_client.download_file(
            '{}'.format(selected_bucket),
            '{}'.format(key_name),
            '{0}\\{1}\\{2}'.format(bucket_folder, nested_dir, key_name.split('/')[-1])
        )

def open_folder_that_contains_s3_bucket():
    subprocess.call('explorer {}'.format(bucket_folder))

if __name__ == "__main__":
    connect_to_aws()
    select_s3_bucket()
    download_s3_bucket()
    open_folder_that_contains_s3_bucket()


import os
import random
import re
import time
from collections import defaultdict
import pandas as pd
from sklearn.utils import shuffle
import subprocess
import math

def augment(replace_sent: str, to_replace_list: list, synonym_dict: dict,rest_row_num,rest_da_num):
    da_sentences = []
    changed_sentence = replace_sent
    da_num = math.ceil(rest_da_num/rest_row_num)
    print(da_num, " replace_sent:", replace_sent)
    if da_num > 0:
        for i in range(da_num):
            for index, replace_word in enumerate(to_replace_list):
                if re.search(replace_word, replace_sent) and i < len(synonym_dict[replace_word]):
                    # if "服装品牌" == replace_word and "组图：成熟美丽阿娇的华丽转身经历了艳照门" in replace_sent:
                    changed_sentence = re.sub(replace_word, synonym_dict[replace_word][i], replace_sent, 1)
            da_sentences.append(changed_sentence)
    # print(len(da_sentences),"da_sentences: ", da_sentences)
    for sen in da_sentences:
        print(sen)
    return da_sentences

def old_augment(replace_sent: str, to_replace_list: list, synonym_dict: dict):
    """
    :param replace_sent: the senences to be replaced
    :param to_replace_list: the words to be replaced in the senences
    :param synonym_dict: a synonym dictionary.
    :return:str
    """
    da_sentences = []

    for replace_word in to_replace_list:
        if re.search(replace_word, replace_sent):
            for synonym_word in random.shuffle(synonym_dict[replace_word]):
                da_sentences.append(re.sub(replace_word, synonym_word, replace_sent, 1))

    # da_sentences = []
    # for replace_word in to_replace_list:
    #     if re.search(replace_word, replace_sent):
    #         for synonym_word in synonym_dict[replace_word]:
    #             da_sentences.append(re.sub(replace_word, synonym_word, replace_sent, 1))
    return da_sentences

def search_replacement(sentence, words):
    searched_list = []
    for word in words:
        if word in sentence:
            searched_list.append(word)
    return searched_list

def get_worked_keys(dir):
    key_files_name = []
    keys_a_file = []
    for root, dirs, files in os.walk(dir):
        for file_name in files:
            if 'keys.csv' in file_name:
                key_files_name.append(dir+'/'+file_name)
                df = pd.read_csv(dir +'/'+ file_name)
                keys_a_file.append(df['keys'].tolist())

    print(key_files_name)
    def chaji_keys(set1, setList):
        set1 = set(set1)
        for a_list in setList:
            print('0 set1=', len(set1), len(set(a_list)))
            set1 -= set(a_list)
            print('1 set1=', len(set1))
        print('---------')
        return list(set1)

    def jiaoji_keys(set1, setList):
        set1 = set(set1)

        for a_list in setList:
            print('0 set1=', len(set1), len(set(a_list)))
            set1 &= set(a_list)
            print('1 set1=', len(set1))
        print('---------')
        return list(set1)

    for i, key_a_file in enumerate(keys_a_file):
        temp_list = keys_a_file.copy()
        temp_list.pop(i)
        keys_work = jiaoji_keys(key_a_file, temp_list)
        df = pd.DataFrame({'keys': keys_work})

        df.to_csv( key_files_name[i].split('.csv')[0] + str(len(keys_work))+"_work.csv", index=False)
        print(key_files_name[i].split('.csv')[0] + str(len(keys_work))+"_work.csv")

def match_keys(file_path,dict_path):
    df = pd.read_csv(file_path + '.csv')
    row_num = len(df)
    print('kda open ', file_path, row_num)
    df_synonyms = pd.read_csv(dict_path)
    df_synonyms.drop(df_synonyms[df_synonyms.final_all_keys.isnull()].index, inplace=True)
    df_synonyms.drop(df_synonyms[df_synonyms.close_words.isnull()].index, inplace=True)
    synonym_dict = defaultdict(list)
    for index, row in df_synonyms.iterrows():
        synonym_dict[row['final_all_keys']] = row['close_words'].split(',')
    hit_keywords = []
    for index, row in df.iterrows():
        sentence = row['content']
        keywords_to_be_replaced = search_replacement(sentence, df_synonyms['final_all_keys'].tolist())
        hit_keywords.extend(keywords_to_be_replaced)

    matched_keys_df = pd.DataFrame({'keys':hit_keywords})
    matched_keys_df.to_csv(file_path + '_keys.csv', index=False)
    print('save ', file_path + '_keys.csv', len(matched_keys_df))

def kda(file_path, dict_path, newsize=-1):

    df = pd.read_csv(file_path+'.csv')
    row_num = len(df)
    da_size = newsize - row_num

    print('kda open ', file_path, row_num, ' ', dict_path)
    df_synonyms = pd.read_csv(dict_path)
    df_synonyms.drop(df_synonyms[df_synonyms.final_all_keys.isnull()].index, inplace=True)
    df_synonyms.drop(df_synonyms[df_synonyms.close_words.isnull()].index, inplace=True)
    synonym_dict = defaultdict(list)
    for index, row in df_synonyms.iterrows():
        synonym_dict[row['final_all_keys']] = row['close_words'].split(',')

    da_sentences = []
    hit_keywords = []

    for index, row in df.iterrows():
        sentence = row['content']
        keywords_to_be_replaced = search_replacement(sentence, df_synonyms['final_all_keys'].tolist())
        hit_keywords.append(keywords_to_be_replaced)

    labels = []
    keys = []
    for index, row in df.iterrows():
        sentence = row['content']
        new_sents = augment(sentence, hit_keywords[index], synonym_dict, row_num-index, da_size)
        da_size -= len(new_sents)
        da_sentences.extend(new_sents)
        for i in range(len(new_sents)):
            labels.append(row['label'])
            keys.append(hit_keywords[index])
        if da_size < 1:
            break
    df_da = pd.DataFrame({"content": da_sentences, "label": labels, "keys": keys})
    new_df = pd.concat([df, df_da])
    new_df.to_csv(file_path+'_kda.csv', index=False)
    return file_path+'_kda.csv'

def kda_single_file(dir, file_path, dict_path, newsize):

    df = pd.read_csv(file_path+'.csv')
    row_num = len(df)
    print('kda open ',file_path,row_num )
    df_synonyms = pd.read_csv(dict_path)
    df_synonyms.drop(df_synonyms[df_synonyms.final_all_keys.isnull()].index, inplace=True)
    df_synonyms.drop(df_synonyms[df_synonyms.close_words.isnull()].index, inplace=True)
    synonym_dict = defaultdict(list)
    for index, row in df_synonyms.iterrows():
        synonym_dict[row['final_all_keys']] = row['close_words'].split(',')

    da_sentences = []
    da_sentences_number = []
    da_total = 0
    hit_keywords = []
    for index, row in df.iterrows():
        sentence = row['content']
        keywords_to_be_replaced = search_replacement(sentence, df_synonyms['final_all_keys'].tolist())
        hit_keywords.extend(keywords_to_be_replaced)
        new_sents = augment(sentence, keywords_to_be_replaced, synonym_dict)
        da_sentences.append(new_sents)
        da_total += len(new_sents)
        da_sentences_number.append(len(new_sents))

    df1 = df_synonyms[df_synonyms.final_all_keys.isin(set(hit_keywords))]
    df1.to_csv(file_path + '_keys.csv', index=False)
    print('save ',file_path + '_keys.csv',len(df1))

    df['da_sentences'] = pd.DataFrame({'da_sentences': da_sentences})
    df['da_sentences_number'] = pd.DataFrame({'da_sentences_number': da_sentences_number})
    # df.to_csv(file_path+'_daDebug.csv', index=False) #for debugging
    selected_precentage = (newsize - row_num) / da_total

    new_sents = []
    new_labels = []
    rest_number = newsize - row_num
    for index, row in df.iterrows():
        new_sents.append(row['content'])#the original sentence
        new_labels.append(row['label'])
        selected_number = int(row['da_sentences_number']*selected_precentage)
        if selected_number < 15 and selected_number>0:
            selected_number += 1

        if rest_number <= selected_number:
            # #print('rest_number=', rest_number, ' selected_number=', selected_number)
            selected_number = rest_number
        for i in range(selected_number):#the generated sentences
            # #print('i=',i,' len(row[da_sentences]=',len(row['da_sentences']))
            new_sents.append(row['da_sentences'][i])
            new_labels.append(row['label'])
        rest_number -= selected_number

    #print('rest_number=', rest_number)
    da_df = pd.DataFrame({'label':new_labels, 'content':new_sents})
    da_df.to_csv(file_path+'_kda.csv', index=False)
    return file_path+'_kda.csv' ,

def merge_csv(files, merge_path):
    if len(files) < 1:
        return None
    dfs = []
    for f in files:
        if f is None:
            return None
        else:
            df = pd.read_csv(f)
            df = shuffle(df)
            dfs.append(df)
    concat_df = pd.concat(dfs, axis=0, ignore_index=True)
    concat_df = shuffle(concat_df)
    concat_df.to_csv(merge_path, index=False)
    return merge_path

def merge_path(path, out_file_name, sub_name_in_file='clean'):
    pd.set_option('display.max_columns', None)
    pd.set_option('display.max_rows', None)
    pd.set_option('max_colwidth', 100)
    # -----------merge all files
    path = path+'/'
    tomerge_files = []
    for root, dirs, files in os.walk(path):
        for file_name in files:
            if sub_name_in_file in file_name:
                tomerge_files.append(path + file_name)
        break
    #print('merge files:',tomerge_files)
    merge_csv(tomerge_files, path + out_file_name)

def augument_data(original_data, da_number):
    """
    :param original_data: a list of orginal data paths    type:list
    :param number: the amount of the data after data augmentation process  type:int
    """
    data_kda_folders = [path+'_kda_'+str(da_number)+'/' for path in original_data]
    refined_keywords_dir = 'data/refine_keywords/'
    time_start = time.time()

    for index, folder_p in enumerate(data_kda_folders):
        # print(index,'   ',folder_p)
        subprocess.getstatusoutput('rm -rf ' + folder_p)
        subprocess.getstatusoutput('cp -rf ' + original_data[index] + ' ' + folder_p)
        subprocess.getstatusoutput('find ' + folder_p + ' -name train.csv |xargs rm')

        for root, dirs_p, files in os.walk(folder_p):
            for dir in dirs_p:
                for root, dirs, files in os.walk(folder_p + dir):
                    for file_name in files:
                        if 'test' not in file_name and 'dev' not in file_name and 'keys' not in file_name:
                            # match_keys( folder_p + dir + '/' + file_name.split('.csv')[0], dict_path )
                            f_name = file_name.split('.csv')[0]
                            dict_path = refined_keywords_dir + dir + '/' + f_name + 'refine.csv'
                            kda(folder_p + dir + '/' + f_name, dict_path, da_number)
                            # return

    print("Augmenting all the data, take times :" + str((time.time() - time_start) / 60) + ' mins')
    # build training set
    for folder_p in data_kda_folders:
        for root, dirs_p, files in os.walk(folder_p):
            for dir in dirs_p:
                merge_path(folder_p + dir, 'train.csv', '_kda.csv')
            break

if __name__ == "__main__":
    #To augment 5000 data
    augument_data(['data/data_500', 'data/data_2000'],5000)
    augument_data(['data/data_2000', 'data/data_2000'], 5000)

    # To augment 10000 data
    #augument_data(['data/data_500', 'data/data_2000'], 10000)


import glob
import sys, os
from collections import defaultdict
import numpy as np

F = glob.glob("results/*.csv")
print(F)

SPK_MOS = defaultdict(list)
ANN_MOS = defaultdict(list)
agg_out_file = "test_file.csv"
AGG = []

# test1,5876-hvd_051.wav," 4 ",C1,23514
for f in F:
    ann = f.split(".")[0]
    print(f)
    input = open(f, "r")
    data = input.read().split("\n")[1:-1]
    input.close()
    print(len(data))
    for line in data:
        items = line.split(",")
        ann = items[0]
        spk = items[1].split("-")[0]
        mos = int(items[2].split(" ")[1])
        fname = items[1]
        SPK_MOS[spk].append(mos)
        ANN_MOS[ann].append(mos)

        newline = ann+","+spk+",ophelia,"+fname+","+str(mos)
        AGG.append(newline)

output = open(agg_out_file, "w")
output.write("\n".join(AGG))
output.close()

for ann,moslist in ANN_MOS.items():
    avg_mos = np.mean(np.array(moslist))
    min_mos = np.amin(np.array(moslist))
    max_mos = np.amax(np.array(moslist))
    std_mos = np.std(np.array(moslist))
    print(ann,avg_mos, std_mos, min_mos, max_mos)

for spk,moslist in SPK_MOS.items():
    avg_mos = np.mean(np.array(moslist))
    print(spk,avg_mos)