#!/usr/bin/python

'''
This is assignment 3, group submission by :
    Student Name : Ravi Mishra Student ID: 21249928
    Student Name : Prasad Deshpande Student ID: 21249530

    Link to Github Repository : https://github.com/ravi3990/ARC.git

'''

import os, sys
import json
import numpy as np
import re
import math

### YOUR CODE HERE: write at least three functions which solve
### specific tasks by transforming the input x and returning the
### result. Name them according to the task ID as in the three
### examples below. Delete the three examples. The tasks you choose
### must be in the data/training directory, not data/evaluation.

'''
solve_d4a91cb9:

Difficulty Level according to us : Easy to Medium

Logic :

The basic logic for solving this problem through the naked eye and to a layman
person would be get the coordinates of the two colours ie sky blue and red.The
values for these in our list is 2 and 8.And then connect the 2 points throug
it's row and column by two lines of yellows till it intersects.

We used the numpy where function to find the coordinates of these points.The main
problem in solving this issue was to find out in what direction the line has to be plotted.
For this we used the if else to determine if blue has higher column value or red.
Based on that we update the corresponding values in the given row and column to 4.

Basic python and numpy used.Was able to solve all inputs and outputs in the json
file successfully.

'''

def solve_d4a91cb9(x):
    #Changing to Numpy array in case the input is a list
    b=np.array(x)
   #Getting the position of 8 and 2 in the array
    eight_pos=np.where(b==8)[0][0],np.where(b==8)[1][0]
    two_pos=np.where(b==2)[0][0],np.where(b==2)[1][0]
    #Changing the column values to 4
    if eight_pos[0]<two_pos[0]:
        for i in range (eight_pos[0]+1,two_pos[0]+1):
            b[i][eight_pos[1]]=4
    elif eight_pos[0]>two_pos[0]:
        for i in range (two_pos[0],eight_pos[0]):
            b[i][eight_pos[1]]=4

    #Changing the row  values to 4
    if eight_pos[1]<two_pos[1]:
        for i in range (eight_pos[1]+1,two_pos[1]):
            b[two_pos[0]][i]=4
    elif eight_pos[1]>two_pos[1]:
        for i in range (two_pos[1]+1,eight_pos[1]+1):
            b[two_pos[0]][i]=4
    return b

'''
solve_f5b8619d:

Difficulty Level according to us : Easy

Logic :

On a first look it looked a little difficult to us.However we could quickly understand
that it is a mirror effect after some basic changes to the inital array.
In the initial array we have to identify the columns where the value is non zero.
We have to change the value of all the rows for that column without touching the original
values to 8.Once it is completed we have to mirror the array along the horizontal
axis once and then the vertical axis once.This can be done interchangeably as well.

Basic python and numpy has been used.

All the test inputs and outputs were successfully validated and passed.

'''

def solve_f5b8619d(x):
    new_arr=np.array(x)
    #Getting the columns where the value is non zero and changing rest of the column values to 8
    for mn in np.where(new_arr!=0)[1]:
        for i in range(new_arr.shape[1]):
            if new_arr[i][mn]==0:
                new_arr[i][mn]=8
     #Mirroring the array horizontally and vertically
    new_arr=np.concatenate((new_arr,new_arr),axis=0)
    new_arr=np.concatenate((new_arr,new_arr),axis=1)
    return new_arr

'''
solve_0a938d79:

Difficulty Level according to us : Easy

We found Task 0a938d79 simple to understand and implement. Looking at task demonstration
we can see alternate coloured line pattern repeats until end of array. If number of rows are
more than the columns then line patterns are repeated until last row. If the number of
columns are more than the rows then the line patterns are repeated until end of last column.
Count for intermediate black rows are dependent on number of columns/rows between given
two colour cells.

Solver function checked shape of array, colours of non-black locations and loop through
the rows/columns until max number of rows/columns reached. Coloured locations/cells are
used as reference to paint the location to create the line in X or Y axis and new array
return after transformation.

All training and test grids solved correctly.

'''

def solve_0a938d79(x):
    '''empty list to collect coloured cells'''
    non_zero = []

    ''' copy input array'''
    new_arr=np.array(x)

    ''' get number of rows are colums with shape'''
    num_rows, num_cols = new_arr.shape

    '''find the non-zero coordinates '''
    non_zero = np.nonzero(new_arr)

    '''find the colour at non-zero coordinates '''
    colour_1 = new_arr[non_zero[0][0], non_zero[1][0]]
    colour_2 = new_arr[non_zero[0][1], non_zero[1][1]]

    '''if number of columns are greater than rows then patterns need to be
    formed in x-axis else in y-axis'''

    if(num_cols > num_rows):
        black_mid_segments = abs(non_zero[1][1] - non_zero[1][0])
        while((non_zero[1][0] or non_zero[1][1]) < num_cols):
            '''fill the colour at non-zero column'''
            for i in range(num_rows):
                if(non_zero[1][0] < num_cols):
                    new_arr[i,non_zero[1][0]] = colour_1
                if(non_zero[1][1] < num_cols):
                    new_arr[i,non_zero[1][1]] = colour_2
            '''increment coloumn index based number of empty col. in pattern'''
            non_zero[1][0] += (black_mid_segments*2)
            non_zero[1][1] += (black_mid_segments*2)
    else:
        black_mid_segments = abs(non_zero[0][0] - non_zero[0][1])
        while((non_zero[0][0] or non_zero[0][1]) < num_rows):
            '''fill the colour at non-zero rows'''
            for i in range(num_cols):
                if(non_zero[0][0] < num_rows):
                    new_arr[non_zero[0][0], i] = colour_1
                if(non_zero[0][1] < num_rows):
                    new_arr[non_zero[0][1], i] = colour_2
            '''increment row index based number of empty col. in pattern'''
            non_zero[0][0] += (black_mid_segments*2)
            non_zero[0][1] += (black_mid_segments*2)

    ''' return the modified array'''
    return new_arr

'''
solve_1a07d186(x)

Difficulty Level according to us : Medium to difficult

We found Task 1a07d186 medium to be difficult. On the first look, task seems easy
i.e. just moving the coloured cells adjacent to respective lines. But it is also important to
identify line as object, move coloured cells in right direction and to remove other
coloured cells which doesn’t have coloured line present. The same logic then needs to be
defined for vertical and horizontal lines (single/multiple).

We decided to implement the solver function with basic logic and with less use of
library APIs. Numpy is used only where necessary, so the code looks big. It identified
colour in array and marked the coordinates. Then identified the line object with traversing
in X/Y axis from edge to edge to confirm the line position. Once all these information
is available, then look for nearest line coordinate for coloured cell and paint next
location in same row/column. The original colour location is then overwritten with black.
In case of cells with colours other than line colour, the location is overwritten with black.

All training and test grids solved correctly.

'''

def solve_1a07d186(x):

    ''' copy input array'''
    new_arr=np.array(x)

    ''' get number of rows are colums with shape'''
    num_rows, num_cols = new_arr.shape

    '''find the non-zero coordinates '''
    non_zero = np.nonzero(new_arr)

    ''' index for checking all coordinates around the colour'''
    Dy = [0, -1, 0, 1]
    Dx = [1, 0, -1, 0]

    ''' empty lists to log if Hlines or Vlines found in image'''
    H_line = []
    V_line = []

    ''' loop to check colour around all identified colour co-ordinates '''
    for x, y  in zip(non_zero[0],non_zero[1]):
        colour = new_arr[x][y]

        for dy, dx in zip(Dy, Dx):
            ''' if same colour found then move either x or y axis to find if it is a line'''
            x_, y_ = x+dx, y+dy

            ''' loop through number of cols or rows to find lines and log into list '''
            if(x_ < num_rows and y_ < num_cols):
                if(colour == new_arr[x_][y_]):
                    if(abs(x_ - x)):
                        #vertical line
                        for k in range(num_rows):
                            if(colour == new_arr[k][y]):
                                V_line.append(y)
                                v_line_flag = True

                    if (abs(y_-y)):
                        #horizontal line
                        for k in range(num_cols):
                            if(colour == new_arr[x][k]):
                                H_line.append(x)
                                v_line_flag = False

    ''' consider only unique items to give exact number of lines and their one
    coordinates location. Other location can be assumed zero as it is a line'''
    H_line = np.unique(H_line)
    V_line = np.unique(V_line)

    ''' loop through all coloured cellls to either associate with line or if
    not then make it zero'''
    for x, y  in zip(non_zero[0],non_zero[1]):
        if(v_line_flag == True):
            ''' if y is not in Vline then proceed '''
            if(y not in V_line):
                colour_flag = False
                colour = new_arr[x][y]

                for i in V_line:
                    if(colour == new_arr[0, i]):
                        if(i < y):
                            new_arr[x,i+1] = colour
                        else:
                            new_arr[x,i-1] = colour

                        new_arr[x][y] = 0
                        colour_flag = True
                ''' if colour cell is not associated with line colour then
                make it zero '''
                if(colour_flag == False ):
                    new_arr[x][y] = 0

        if(v_line_flag == False):
            '''if x is not in Hline then proceed '''
            if(x not in H_line):
                colour_flag = False
                colour = new_arr[x][y]

                for i in H_line:
                    if(colour == new_arr[i, 0]):
                        if(i < x):
                            new_arr[i+1, y] = colour
                        else:
                            new_arr[i-1, y] = colour

                        new_arr[x][y] = 0
                        colour_flag = True
                ''' if colour cell is not associated with line colour then
                make it zero '''
                if(colour_flag == False ):
                    new_arr[x][y] = 0

    ''' return the modified array'''
    return new_arr

'''
def solve_00d62c1b(x)

Difficulty Level according to us : Medium to difficult

We found task 00d62c1b medium to be difficult. The challenge was in finding the enclosed
shape to fill the space with yellow colour. The logic which is applied here is same
as previous task 1a07d186, instead of lines, here is to identify the space enclosed
with green colour.

Same logic of x, y location +/-1 for checking all coordinates around the coloured
cell used. Code optimized with the use of list as queue and multiple conditions in loops.
‘Searched’ array created with padding of rows and columns. Based on search the true/False
marked and resultant used for filling yellow colour.
The in-line comments give more details of logic.

All training and test grids solved correctly.

'''

def solve_00d62c1b(x):

    ''' copy input array'''
    new_arr=np.array(x)

    ''' get number of rows are colums with shape'''
    num_rows, num_cols = new_arr.shape

    green = 3
    yellow = 4

    ''' index for checking all coordinates around the colour'''
    Dy = [0, -1, 0, 1]
    Dx = [1, 0, -1, 0]

    '''
    https://numpy.org/doc/stable/reference/generated/numpy.pad.html

    ((1,1),(1,1)) -Number of values padded to the edges of each axis. ((before_1, after_1)
    pads with 'Constant' value of zero [False]

    '''

    arr_padded = np.pad(new_arr, ((1,1),(1,1)), "constant", constant_values=0)

    '''
    input array padded on x and y axis at the edge i.e. num_col and num_rows
    increased by 2 and created searched array with default False in each element'''
    search_empty_sq = np.zeros(arr_padded.shape, dtype=bool)
    search_empty_sq[0, 0] = True

    queue = [(0, 0)]
    while queue:
        ''' take each element/cell from matrix'''
        j, i = queue.pop()

        ''' loop for near by coordinates in x, y axis '''
        for dy, dx in zip(Dy, Dx):
            y_, x_ = j+dy, i+dx

            if not 0 <= y_ < num_rows+2 or not 0 <= x_ < num_cols+2:
                ''' skip the next logic for edge elements/cells'''
                continue
            ''' in case of identical elements change value at searched to True
                else maintain the value at False. False indicates yellow colour
                to add
            '''
            if not search_empty_sq[y_][x_] and arr_padded[y_][x_]==0:
                '''add to queue for next operation '''
                queue.append((y_, x_))
                search_empty_sq[y_, x_] = True

    '''remove the padding'''
    res = search_empty_sq[1:-1, 1:-1]

    ''' check the green and keep it as it is'''
    res |= new_arr==green

    ''' invert the searched results and fill true with yellow '''
    new_arr[~res] = yellow

    ''' return the modified array'''
    return new_arr

'''
def solve_d0f5fe59(x)

Difficulty level according to us : Medium

Logic:
The idea behind solving this problem was to get the number of islands within a
particular array.This was done through a depth first search.Once a particulat
island is visited it is not counted twice.For this we created an internal functions
called dfs.

After iterating through the entire array we found the number of islands.We then
created a numpy zero array with the same dimensions as the number of islands.
We then update the diagonal of the array with the value 8.

We were able to succsfully validate and test all the inputs with their given outputs
as in the json file.

'''

def solve_d0f5fe59(x):
    row=len(x)
    col=len(x[0])
    count=0
   #Creating a function for Depth first search for finding the islands
    def dfs(a,row,col,m,n):
        if a[m][n]==0:
            return
        a[m][n]=0

        if m!=0:
            dfs(a,row,col,m-1,n)

        if m!=row-1:
            dfs(a,row,col,m+1,n)

        if n!=0:
            dfs(a,row,col,m,n-1)
        if n!=col-1:
            dfs(a,row,col,m,n+1)
            #Iterating to count the number of islands
    for i in range(row):
        for j in range(col):
            if x[i][j]==8:
                dfs(x,row,col,i,j)
                count+=1
     #Creating a zero numpy array with the same dimensions as the number of islands
    b=np.zeros((count, count), int)
    #Filling the diagonal with the value 8
    np.fill_diagonal(b,8)

    return b

'''
def solve_4093f84a(x)

Difficulty Level  according to us : Medium to Difficult

Logic :
4093f84a.json looked to be simple on the testing interface initially.Logically
we needed to find the rectangle with 5's and then determine whether it was a horizontal
one or a vertical one.After that close to the the rectangle  we had to fill the 5 in
the respective rows or columns.

We tried to have a simple approach of finding the bigger island with the rectangle first.
After that we made all the other positions as zero.We noted the positions of the other
values.And then added them to the rectangle in their respective rows and columns.

The given code use basic python and numpy functions.
While testing this it passed for the test demonstration 1.However there was a
slight difference in demosntraions 2 and 3 as there were multiple non 5 values in the
same row or column.This made it to not completely satisfy the outputs for 2 and 3 and test input grid.

The idea to solve this would be to get the positions ond the number of such colours
in a particular row or column.Once we get the number we can add to the same side of
the rectangle the cells and this would result in the success of the function

'''

def solve_4093f84a(x):
    x=np.array(x)
    #Getting positions where the value is non zero
    mn=np.where(x!=0)
    #Getting positions where the value is five
    yz=np.where(x==5)
    emp_lst=[]
    #Adding to an empty list the positions of the non 5 and non zero values
    for i in range(len(mn[0])):
        if x[mn[0][i]][mn[1][i]]!=5:
            emp_lst.append([mn[0][i],mn[1][i]])
    #Making all the non island positions 0 before actioning
    for m in emp_lst:
        x[m[0],m[1]]=0
    #Action to alter the value to 5 for the respective postions near to the island
    if np.amin(yz[1])==0:
        for m in emp_lst:
            if m[0]<np.amin(yz[0]):
                x[np.amin(yz[0])-1][m[1]]=5
            else :
                x[np.amax(yz[0])+1][m[1]]=5
    else :
        for m in emp_lst:
            if m[1]<np.amin(yz[1]):
                x[m[0]][np.amin(yz[1])-1]=5
            else :
                x[m[0]][np.amax(yz[1])+1]=5
    return x

'''
def solve_5ad4f10b(x)

Difficulty Level according to us : Difficult

Logic:

For solving 5ad4f10b.json we tried to look at various ways for coming up.It seemed
a little difficult.The main challenge was to find the area for the section which was
to be contracted and then the values updated with the other colour outliers.

Following is the approach we followed for this section :

1.Identify the n*n squares(start with 3) in the given array,their coordinates as
well and store their values in separate lists.
2.Get the max and min values of the rows and column numbers for the areas containing squares.
3.Create a new array with only the rows and columns which  was obtained from Step 2
4.Clear any other colour values than 0 and the squares and convert it to 0.
5.Get the other colour value from the original array and create a new array with the specific positions.

We have used basic numpy and math library to get this output.

The one limitation to this part of the code is that it is currently useful for
3*3 squares only(Training Input and output 2 currently. We can use a similar version of
the same code by altering the for loops a little to make it working for 4*4 squares as well.
Due to timing limitations we have not tried it. However we believe the logic would be near
about the same and would work for other training and test grids as well with slight modifications.
'''

def solve_5ad4f10b(x):
    a=np.array(x)
    row,colmn=a.shape[0],a.shape[1]

     #Code to get the 3*3 squares inside the array

    emp_lst1=[]
    range_lst1=[]
    #comp=0
    for i in range(row):
        for j in range(colmn):
            emp_lst=[]
            range_lst=[]
            comp=0
            if i-1>0 and j-1>0 and i+1<row and j+1<colmn:
                for m in range(i-1,i+2):
                    for n in range(j-1,j+2):
                        emp_lst.append(a[m][n])
                        range_lst.append ((m,n))

            if len(emp_lst)>0 and emp_lst[0]!=0:
                comp=1
                #print (emp_lst )
                for mn in emp_lst:
                    if emp_lst[0]!=mn:
                        comp+=1
            #print(comp)
            if comp==1 and len(emp_lst)>0:
                #print(emp_lst)
                #print(range_lst)
                emp_lst1.append(emp_lst)
                range_lst1.append(range_lst)

    #Getting the minimum and maximum section of the array to be cut

    min_row=range_lst1[0][0][0]

    for m in range_lst1:
        for n in m:
            if n[0]<min_row:
                min_row=n[0]

    min_col=range_lst1[0][0][1]

    for m in range_lst1:
        for n in m:
            if n[1]<min_col:
                min_col=n[1]
    max_row=range_lst1[0][0][0]

    for m in range_lst1:
        for n in m:
            if n[0]>max_row:
                max_row=n[0]

    max_col=range_lst1[0][0][1]

    for m in range_lst1:
        for n in m:
            if n[1]>max_col:
                max_col=n[1]

    new_arr=a[min_row:max_row+1,min_col:max_col+1]
    #Clear the array for any oher number which are outliers
    new_row,new_col=new_arr.shape[0],new_arr.shape[1]

    for i in range(new_row):
        for j in range(new_col):
            if new_arr[i][j]!=emp_lst1[0][0]:
                new_arr[i][j]=0

    new_mod_arr=np.zeros((int(math.sqrt(new_row)),int(math.sqrt(new_col))))
    row_alt,col_alt=new_mod_arr.shape[0],new_mod_arr.shape[1]

    #Getting the colour value which has to be updated
    non_zero_val_loc=np.where(a!=0)

    for m in non_zero_val_loc[0]:
        for n in non_zero_val_loc[1]:
            if a[m][n]!=emp_lst1[0][0]:
                val_to_update=a[m][n]

    #Updating the new modified array with the intended values
    for i in range(0,new_row,3):
        for j in range(0,new_col,3):
            if new_arr[i][j]!=0:
                new_i=int(i/3)
                new_j=int(j/3)
            #print (new_i,new_j)
                new_mod_arr[new_i,new_j]=val_to_update
    return new_mod_arr

'''

A short summary/reflection, commenting on the Python features and libraries you used in the solve *

In first semester of MSc. while learning and working on assignments we could see maturity
of available ML packages which can trained on different kind of tasks e.g., NLP task for
suggestion mining or regression tasks for prediction of home prices etc. but while working
with ARC, we realized that we’re still very far away from building a human like intelligence.
All the tasks (.JSON files) when looked in browser, one can identify the patterns in seconds
just looking at few samples, but it is not easy for ML/programming to generalize this type
of intelligence. We didn’t find much help on generalised pattern recognition available,
and ARC is an upcoming new topic in field of AI. We both found it is super interesting
and addictive i.e., we initially though to just complete 3-4 patterns and submit the assignment
but end up coding 8-9 patterns!

We did not use any ML package or specific libraries but used Numpy and pure python to
translate our understanding of patterns into code. We found that, in future there can
be few common functions/libraries be made to recognize the patterns e.g., find specific
shape objects or enclosed spaces, highlight repetitive patterns, symmetry etc. or correlate
the colours in base image to transformed image. These functions in addition to math
libraries can be then basis for creating hypothesise in terms of hyper parameters or
support functions. We still have limited understanding of this topic and acknowledge
that writing the code to solve all sorts of problems i.e., generalisation is not easy
task.

Commonalities between the problems :

->Pattern Recognition is very important.
->Many problems include finding the position of enclosed areas and islands.
->In many problems we had dimensionality reshaping as well as revaluation of the array.
->Most of the problems can be solved via if else loops or basic libraray functions.
  However as mentioned before  there is no one off solution to all problems

Github has been thoroughly used by us for core sharing and working as a team of 2.
Readme file has been altered as well for reflecting the ARC understanding.

As mentioned by François Chollet in his paper 'The Measure of Intelligence' the
solutions can be generalised only to a limited extent because of some of the limitations
which we could see while solving our given problems.The major limitations were a
generalisation of the problem and limited set of inputs and outputs for training
on the same problem. We could only test what was given to use.We are however aware
that there will be many variations in the given formations.

No doubt there are still many leaps to be taken by Arificial Intelligence!

'''

def main():
    # Find all the functions defined in this file whose names are
    # like solve_abcd1234(), and run them.

    # regex to match solve_* functions and extract task IDs
    p = r"solve_([a-f0-9]{8})"
    tasks_solvers = []
    # globals() gives a dict containing all global names (variables
    # and functions), as name: value pairs.
    for name in globals():
        m = re.match(p, name)
        if m:
            # if the name fits the pattern eg solve_abcd1234
            ID = m.group(1) # just the task ID
            solve_fn = globals()[name] # the fn itself
            tasks_solvers.append((ID, solve_fn))

    for ID, solve_fn in tasks_solvers:
        # for each task, read the data and call test()
        directory = os.path.join("..", "data", "training")
        json_filename = os.path.join(directory, ID + ".json")
        data = read_ARC_JSON(json_filename)
        test(ID, solve_fn, data)

def read_ARC_JSON(filepath):
    """Given a filepath, read in the ARC task data which is in JSON
    format. Extract the train/test input/output pairs of
    grids. Convert each grid to np.array and return train_input,
    train_output, test_input, test_output."""

    # Open the JSON file and load it
    data = json.load(open(filepath))

    # Extract the train/test input/output grids. Each grid will be a
    # list of lists of ints. We convert to Numpy.
    train_input = [np.array(data['train'][i]['input']) for i in range(len(data['train']))]
    train_output = [np.array(data['train'][i]['output']) for i in range(len(data['train']))]
    test_input = [np.array(data['test'][i]['input']) for i in range(len(data['test']))]
    test_output = [np.array(data['test'][i]['output']) for i in range(len(data['test']))]

    return (train_input, train_output, test_input, test_output)

def test(taskID, solve, data):
    """Given a task ID, call the given solve() function on every
    example in the task data."""
    print(taskID)
    train_input, train_output, test_input, test_output = data
    print("Training grids")
    for x, y in zip(train_input, train_output):
        yhat = solve(x)
        show_result(x, y, yhat)
    print("Test grids")
    for x, y in zip(test_input, test_output):
        yhat = solve(x)
        show_result(x, y, yhat)

def show_result(x, y, yhat):
    print("Input")
    print(x)
    print("Correct output")
    print(y)
    print("Our output")
    print(yhat)
    print("Correct?")
    if y.shape != yhat.shape:
        print(f"False. Incorrect shape: {y.shape} v {yhat.shape}")
    else:
        print(np.all(y == yhat))

if __name__ == "__main__": main()


# coding: utf-8

# # Step 2 - Test The Model
#
# In this notebook, we will use the model that we trained in Step 1 to drive the car around in AirSim. We will make some observations about the performance of the model, and suggest some potential experiments to improve the model.
#
# First, let us import some libraries.

# In[ ]:

from keras.models import load_model
import sys
import numpy as np
import glob
import os

if ('../../PythonClient/' not in sys.path):
    sys.path.insert(0, '../../PythonClient/')
from AirSimClient import *

# << Set this to the path of the model >>
# If None, then the model with the lowest validation loss from training will be used
MODEL_PATH = None

if (MODEL_PATH == None):
    models = glob.glob('model/models/*.h5')
    best_model = max(models, key=os.path.getctime)
    MODEL_PATH = best_model

print('Using model {0} for testing.'.format(MODEL_PATH))

# Next, we'll load the model and connect to AirSim Simulator in the Landscape environment. Please ensure that the simulator is running in a different process *before* kicking this step off.

# In[3]:

model = load_model(MODEL_PATH)

client = CarClient()
client.confirmConnection()
client.enableApiControl(True)
car_controls = CarControls()
print('Connection established!')

# We'll set the initial state of the car, as well as some buffers used to store the output from the model

# In[4]:

car_controls.steering = 0
car_controls.throttle = 0
car_controls.brake = 0

image_buf = np.zeros((1, 59, 255, 3))
state_buf = np.zeros((1,4))

# We'll define a helper function to read a RGB image from AirSim and prepare it for consumption by the model

# In[5]:

def get_image():
    image_response = client.simGetImages([ImageRequest(0, AirSimImageType.Scene, False, False)])[0]
    image1d = np.fromstring(image_response.image_data_uint8, dtype=np.uint8)
    image_rgba = image1d.reshape(image_response.height, image_response.width, 4)

    return image_rgba[76:135,0:255,0:3].astype(float)

# Finally, a control block to run the car. Because our model doesn't predict speed, we will attempt to keep the car running at a constant 5 m/s. Running the block below will cause the model to drive the car!

# In[ ]:

while (True):
    car_state = client.getCarState()

    if (car_state.speed < 5):
        car_controls.throttle = 1.0
    else:
        car_controls.throttle = 0.0

    image_buf[0] = get_image()
    state_buf[0] = np.array([car_controls.steering, car_controls.throttle, car_controls.brake, car_state.speed])
    model_output = model.predict([image_buf, state_buf])
    car_controls.steering = round(0.5 * float(model_output[0][0]), 2)

    print('Sending steering = {0}, throttle = {1}'.format(car_controls.steering, car_controls.throttle))

    client.setCarControls(car_controls)

# ## Observations and Future Experiments
#
# We did it! The car is driving around nicely on the road, keeping to the right side for the most part, carefully navigating all the sharp turns and instances where it could potentially go off the road. However, you would immediately notice a few other things. Firstly, the motion of the car is not smooth, especially on those bridges. Also, if you let the model running for a while (a little more than 5 minutes), you will notice that the car eventually veers off the road randomly and crashes. But that is nothing to be disheartened by! Keep in mind that we have barely scratched the surface of the possibilities here. The fact that were able to have the car learn to drive around almost perfectly using a very small dataset is something to be proud of!
#
# > **Thought Exercise 2.1**:
# As you might have noticed, the motion of the car is not very smooth on those bridges. Can you think of a reason why it is so? Can you use one of the techniques we described in Step 0 to fix this?
#
# > ** Thought Exercise 2.2**:
# The car seems to crash when it tries to climb one of those hills. Can you think of a reason why? How can you fix this? (Hint: You might want to take a look at what the car is seeing when it is making that ascent)
#
# AirSim opens up a world of possibilities. There is no limit to the new things you can try as you train even more complex models and use other learning techniques. Here are a few immediate things you could try that might require modifying some of the code provided in this tutorial (including the helper files) but won't require modifying any Unreal assets.
#
# > ** Exploratory Idea 2.1**:
# If you have a background in Machine Learning, you might have asked the question: why did we train and test in the same environment? Isn't that overfitting? Well, you can make arguments on both sides. While using the same environment for both training and testing might seem like you are overfitting to that environment, it can also be seen as drawing examples from the same probability distribution. The data used for training and testing is not the same, even though it is coming from the same distribution. So that brings us to the question: how will this model fare in a different environment, one it hasn't seen before?
# This current model will probably not do very well, given that the other available environments are very different and contain elements that this model has never seen before (intersections, traffic, buildings etc.). But it would be unfair to ask this model to work well on those environments. Think of it like a human who has only ever driven in the mountains, never seen other cars or intersections in their entire life, is suddenly asked to drive in a city. How well do you think they will fare?
# The opposite case should be interesting though. Does training on data collected from one of the city environments generalize easily to driving in the mountains? Try it yourself to find out.
#
# > ** Exploratory Idea 2.2**:
# We formulated this problem as a regression problem - we are predicting a continuous valued variable. Instead, we could formulate the problem as a classification problem. More specifically, we could define buckets for the steering angles (..., -0.1, -0.05, 0, 0.05, 0.1, ...), bucketize the labels, and predict the correct bucket for each image. What happens if we make this change?
#
# > ** Exploratory Idea 2.3**:
# The model currently views a single image and a single state for each prediction. However, we have access to historical data. Can we extend the model to make predictions using the previous N images and states (e.g. given the past 3 images and past 3 states, predict the next steering angle)? (Hint: This will possibly require you to use recurrent neural network techniques)
#
# > ** Exploratory Idea 2.4**:
# AirSim is a lot more than the dataset we provided you. For starters, we only used one camera and used it only in RGB mode. AirSim lets you collect data in depth view, segmentation view, surface normal view etc for each of the cameras available. So you can potentially have 20 different images (for 5 cameras operating in all 4 modes) for each instance (we only used 1 image here). How can combining all this information help us improve the model we just trained?


"""

Experiment specifications:
an experiment is defined by train,test dataset pair,
each dataset is loaded from graphical_models/datasets.
Authors: kkorovin@cs.cmu.edu

"""

import os
import numpy as np

from graphical_models import BinaryMRF
from inference import get_algorithm
from graphical_models.data_gen import struct_names
from constants import *

# Give specs in form structure->size
# when used for train, the same is model name
data_specs = {
    "debug":
            {"star": [5],
              "fc":   []},
    "larger_debug":
            {"star": [10],
              "fc":   []},
}

# add simple datasets
data_specs.update({struct+"_small": {struct: [9]} for struct in struct_names})
assert "star_small" in data_specs

# add compound datasets
data_specs.update({struct+"_medium": {struct: [15,16,17]} for struct in struct_names})
data_specs.update({"trees_medium": {"star": [15, 16, 17],
                                    "path": [15, 16, 17],
                                    },
                    "conn_medium": {"bipart": [15, 16, 17],
                                    # "tripart": [15, 16, 17],
                                    "fc": [15, 16, 17],
                                    },
                  })
data_specs.update({"grid_large":{"grid":[49]},
                  "path_large": {"path":  [9,10,100]},
                  "fc_large": {"fc": [15,16,17]},
                  "barbell_large": {"barbell": [15,16,17]},
                  "ladder_large": {"ladder": [15,16,17]},
                  "random_tree_large": {"random_tree": [15,16,17]},
                  "wheel_large": {"wheel": [15,16,17,100]},
                 })

# Add experiments for part 2: Trees+BP
data_specs.update({"trees_approx": {"random_tree":  [100]},
                 })

# Add experiments for part 2: NonTrees+MCMC
data_specs.update({"nontrees_approx":
                        {"barbell":  [100],
                        "fc":  [100]},
                    "barbell_approx":
                        {"barbell":  [100]},
                    "fc_approx":
                        {"fc":  [100]}
                 })

# Data loading ----------------------------------------------------------------
def get_dataset_by_name(specs_name, data_dir, mode=None):
    """
    Assumes graphs live as
    graphical_models/datasets/{train/val/test}  <-- data_dir
                                    |-- star/
                                    |    |-  9/<file1.npy>, <file2.npy> ...
                                    |    |- 10/
                                         |- 11/
                                   ...  ...
    Loads all graphs of given size and structure,
    this needs to be updated in the future
    (so that we can train and test on the same structures)

    Arguments:
        specs_name - key to the data_specs dictionary
        data_dir - train or test directory
        mode - map or marginal
    """
    if specs_name not in data_specs:
        raise ValueError("Specification {} not supported".format(specs_name))
    specs = data_specs[specs_name]
    graphs = []
    for struct in specs:
        size_list = specs[struct]
        for size in size_list:
            # go to specified dir, load and append
            directory = os.path.join(data_dir, struct, str(size))

            for filename in os.listdir(directory):
                if filename.endswith(".npy"):
                    path_to_graph = os.path.join(directory, filename)
                    data_dict = np.load(path_to_graph, allow_pickle=True)[()]  # funny indexing
                    graph = BinaryMRF(data_dict["W"], data_dict["b"])
                    graph.set_ground_truth(marginal_est=data_dict["marginal"],
                                           map_est=data_dict["map"])
                    graph.struct = struct
                    graphs.append(graph)

    if mode is not None:
        graphs = [g for g in graphs if getattr(g, mode) is not None]
    print("Loaded {} graphs".format(len(graphs)))
    return graphs

# Some simple checks ----------------------------------------------------------
if __name__ == "__main__":
    train_data = get_dataset_by_name("debug")
    print(train_data[0])
    print("W, b:", train_data[0].W, train_data[0].b)
    print("Marginals:", train_data[0].marginal)
    print("MAP:", train_data[0].map)


# At a basic level, Pandas objects can be thought of as enhanced versions of
# NumPy structured arrays in which the rows/cols are ID'd with labels instead of int indices

# Pandas provides many useful tools, methods, and functionality on top of basic data structures,
# but first you must understand the structures (objects)

# three fundamental Pandas data structures:
    # Series
    # DataFrame
    # Index

import numpy as np
import pandas as pd

##############################
### the Pandas Series object
# a Series is a one-dimensional array of indexed data
# can construct as follows:
data = pd.Series([0.25, 0.5, 0.75, 1.0])
data

# as seen in output, Series wraps both sequence of vals and sequence of indices
# can access with 'values' and 'index' attributes

# 'values' are simply a NumPy array:
data.values

# 'index' is an array-like object of type pd.Index:
data.index

# also like NP array, data can be accessed via associated index:
data[1]
data[1:3]

## 'Series' as a generalized NumPy array
# diff b/w series and np arr:
# presence of index:
    # NP arr has an implicitly defined integer index
    # PD series has an explicitly defined index

# we can make the explicit index a value of any type (not just int):
data = pd.Series([0.25, 0.5, 0.75, 1.0], index=['a', 'b', 'c', 'd'])
data
data['b']

# or make the indeces non-sequential:
data = pd.Series([0.25, 0.5, 0.75, 1.0], index=['2', '3', '5', '7'])
data
data[5]

## 'Series' as a specialized dictionary
# It's more accurrate to think of Series as a special Python dictionary
    # dict maps arbitrary keys to arbitrary values
    # series maps typed keys to a set of typed values

# the typing is important: the static typing is why PD arrs are more efficient than dicts
    # NP arrays > Python lists
    # PD Series > Python dicts

# an example to clarify the relationship --- can create PD Series from dict
population_dict = {'California': 38332521,
                   'Texas': 26448193,
                   'New York': 19651127,
                   'Florida': 19552860,
                   'Illinois': 12882135}
population = pd.Series(population_dict)
population

# can then perform typical dict-style item access on the new PD Series:
population['California']

# additionally, Series supports array-style slicing (while dicts do NOT)
population['California':'Illinois']

## Constructing Series objects
# all constructors so far have been a version of following statement:
pd.Series(data, index='index')

# data can be a NP Array, 'index' defaults to int sequence
pd.Series([2, 4, 6])

# data can be a scalar - behavior is: scaled to fill specified indeces
pd.Series(5, index=[100, 200, 300])

# data can be a dictionary - 'index' defaults to sorted dictionary keys:
pd.Series({2:'a', 1:'b', 3:'c'})
    # NOTE: my iPython terminal is not displaying 1, 2, 3 as book shows.
    #       it shows in the order entered, even when stored in var.

# can set index explicitly for dict if you want, but it's a bit odd to say the least
pd.Series({2:'a', 1:'b', 3:'c'}, index=[3, 2])

##############################
### the Pandas DataFrame object
# If   'Series'    is an analog of 1D array w/ flexible indeces
# Then 'DataFrame' is an analog of 2D array w/ flexible row/col indeces

# i.e. if you think of
    # 2d array as ordered sequence of aligned (sharing index) 1d columns
# then
    # DataFrame as ordered sequence of aligned Series objects

# Demonstration: new Series lisiting area of 5 states mentioned prior
area_dict = {'California': 423967, 'Texas': 695662, 'New York': 141297,
             'Florida': 170312, 'Illinois': 149995}
area = pd.Series(area_dict)
area

# Can use this in conjunction with "popularion" series from before
states = pd.DataFrame({'population': population, 'area': area})
states

# like Series, DataFrame has an index attribute that gives access to ind labels
states.index

# Additionally, DataFrame has a 'columns' attribute, which is an Index object
# containing column labels
states.columns

## DataFrame as a specialized dictionary
# Similarly, we can think of DataFrame as a specialized dictionary
    # Dictionary maps keys to values
    # DataFrame maps column name to a Series of column data

# demonstrated by asking for 'area' attribute, which returns the Series obj
states['area']

# sticking point! in a 2D NP array, data[0] returns first ROW
#                 in a DataFrame, data['col0'] returns first COLUMN
# thus, we prefer to think of DataFrames as generalized dicts for the most part

## Constructing DataFrame objects

# from a single Series object: (i.e. 1-column DataFrame)
pd.DataFrame(population, columns=['population'])

# from a list of dicts (using a list comprehension to create data)
data = [{'a': i, 'b': 2 * i}
         for i in range(3)]
pd.DataFrame(data)

# **NOTE: if keys in dict are missing, PD fills in with "NaN" vals **
pd.DataFrame([{'a': 1, 'b': 2}, {'b': 3, 'c': 4}])

# from a dictionary of Series objects
pd.DataFrame({'population': population, 'area': area})

# from a 2d NP array (if omitting col names, int indeces used for each)
pd.DataFrame(np.random.rand(3, 2),
             columns=['foo', 'bar'], index=['a', 'b', 'c'])

# from a NP structured array
A = np.zeros(3, dtype=[('A', 'i8'), ('B', 'f8')])
A
pd.DataFrame(A)

##############################
### The Pandas Index object
# In both 'Series' and 'Dataframes', we have explicit 'Index' to reference/modify

# Index object is an interesting structure
# can be thought of as either an immutable array or an ordered set
    # (technically multi-set due to repeated vals)

# Consequences: certain operations available on Index objects
ind = pd.Index([2, 3, 5, 7, 11])
ind

## Index as immutable array
# like array, can use standard Python indexing notation to get vals/slices
ind[1]
ind[::2]

# similar attributes familiar to NP arrs
print(ind.size, ind.shape, ind.ndim, ind.dtype)

# as they are immutable, we cannot modify via normal shorthand
ind[1] = 0      # produces runtime error 'Index does not support mutable operations'

## Index as ordered set
# PD objects are designed to facilitate operations such as joins across datasets
# thus, set arithmetic is often very useful

# Index object follows many conventions used by pythons built-in "set" data structure
# unions, intersections, differences, and other combinations can be computed:
indA = pd.Index([1, 3, 5, 7, 9])
indB = pd.Index([2, 3, 5, 7, 11])

indA & indB         # intersection
indA | indB         # union
indA ^ indB         # symmetric difference

# can also equivalently access via object methods if you prefer
indA.intersection(indB)


#!/usr/bin/env python
# coding: utf-8

# In[113]:

# Filtering out the warnings

import warnings

warnings.filterwarnings('ignore')

# In[114]:

# Importing the required libraries

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# # <font color = blue> IMDb Movie Case Study </font>
#
#

# ##  Task 1: Reading the data

# - ### Subtask 1.1: Read the Movies Data.
#
# Read the movies data file provided and store it in a dataframe `movies`.

# In[115]:

# Read the csv file using 'read_csv'. Please write your dataset location here.

inp=pd.read_csv('Movie_IMDB_Data.csv')
inp

# - ###  Subtask 1.2: Inspect the Dataframe
#
# Inspect the dataframe for dimensions, null-values, and summary of different numeric columns.

# In[116]:

# Check the number of rows and columns in the dataframe

inp.shape

# In[117]:

# Check the column-wise info of the dataframe

inp.info()

# In[118]:

# Check the summary for the numeric columns

inp.describe(include='all')

# ## Task 2: Data Analysis
#
# Now that we have loaded the dataset and inspected it, we see that most of the data is in place. As of now, no data cleaning is required, so let's start with some data manipulation, analysis, and visualisation to get various insights about the data.

# -  ###  Subtask 2.1: Reduce those Digits!
#
# These numbers in the `budget` and `gross` are too big, compromising its readability. Let's convert the unit of the `budget` and `gross` columns from `$` to `million $` first.

# In[119]:

# Divide the 'gross' and 'budget' columns by 1000000 to convert '$' to 'million $'

inp['Gross']= inp['Gross'].floordiv(1000000)
inp['budget']=inp['budget'].floordiv(1000000)

# -  ###  Subtask 2.2: Let's Talk Profit!
#
#     1. Create a new column called `profit` which contains the difference of the two columns: `gross` and `budget`.
#     2. Sort the dataframe using the `profit` column as reference.
#     3. Extract the top ten profiting movies in descending order and store them in a new dataframe - `top10`.
#     4. Plot a scatter or a joint plot between the columns `budget` and `profit` and write a few words on what you observed.
#     5. Extract the movies with a negative profit and store them in a new dataframe - `neg_profit`

# In[120]:

# Create the new column named 'profit' by subtracting the 'budget' column from the 'gross' column
inp['profit']=inp['Gross']-inp['budget']

# In[121]:

# Sort the dataframe with the 'profit' column as reference using the 'sort_values' function. Make sure to set the argument
#'ascending' to 'False'

inp.sort_values(by='profit',inplace=True,ascending=False,ignore_index=True)

# In[122]:

# Get the top 10 profitable movies by using position based indexing. Specify the rows till 10 (0-9)

inp1=inp[0:10]
inp1

# In[123]:

#Plot profit vs budget

sns.barplot(data=inp1,x='profit',y='budget')

# The dataset contains the 100 best performing movies from the year 2010 to 2016. However, the scatter plot tells a different story. You can notice that there are some movies with negative profit. Although good movies do incur losses, but there appear to be quite a few movie with losses. What can be the reason behind this? Lets have a closer look at this by finding the movies with negative profit.

# In[124]:

#Find the movies with negative profit

inp[inp['profit']<0]

# **`Checkpoint 1:`** Can you spot the movie `Tangled` in the dataset? You may be aware of the movie 'Tangled'. Although its one of the highest grossing movies of all time, it has negative profit as per this result. If you cross check the gross values of this movie (link: https://www.imdb.com/title/tt0398286/), you can see that the gross in the dataset accounts only for the domestic gross and not the worldwide gross. This is true for may other movies also in the list.

# - ### Subtask 2.3: The General Audience and the Critics
#
# You might have noticed the column `MetaCritic` in this dataset. This is a very popular website where an average score is determined through the scores given by the top-rated critics. Second, you also have another column `IMDb_rating` which tells you the IMDb rating of a movie. This rating is determined by taking the average of hundred-thousands of ratings from the general audience.
#
# As a part of this subtask, you are required to find out the highest rated movies which have been liked by critics and audiences alike.
# 1. Firstly you will notice that the `MetaCritic` score is on a scale of `100` whereas the `IMDb_rating` is on a scale of 10. First convert the `MetaCritic` column to a scale of 10.
# 2. Now, to find out the movies which have been liked by both critics and audiences alike and also have a high rating overall, you need to -
#     - Create a new column `Avg_rating` which will have the average of the `MetaCritic` and `Rating` columns
#     - Retain only the movies in which the absolute difference(using abs() function) between the `IMDb_rating` and `Metacritic` columns is less than 0.5.
#     - Sort these values in a descending order of `Avg_rating` and retain only the movies with a rating equal to or greater than `8` and store these movies in a new dataframe `UniversalAcclaim`.
#

# In[125]:

# Change the scale of MetaCritic

inp['MetaCritic']=inp['MetaCritic']/10
inp['MetaCritic']

# In[126]:

# Find the average ratings

inp['Avg_rating']=inp[['MetaCritic','IMDb_rating']].mean(axis=1)

# In[127]:

#Sort in descending order of average rating
inp.sort_values(by='Avg_rating',inplace=True,ascending=False,ignore_index=True)
inp

# In[128]:

# Find the movies with metacritic-Imdb rating < 0.5 and also with an average rating of >= 8 (sorted in descending order)
UniversalAcclaim=inp[abs(inp['MetaCritic']-inp['IMDb_rating'])<0.5]
UniversalAcclaim[(abs(inp['MetaCritic']-inp['IMDb_rating'])<0.5) & (inp['Avg_rating']>=8)].sort_index()[0:5]

# **`Checkpoint 2:`** Can you spot a `Star Wars` movie in your final dataset?

# - ### Subtask 2.4: Find the Most Popular Trios - I
#
# You're a producer looking to make a blockbuster movie. There will primarily be three lead roles in your movie and you wish to cast the most popular actors for it. Now, since you don't want to take a risk, you will cast a trio which has already acted in together in a movie before. The metric that you've chosen to check the popularity is the Facebook likes of each of these actors.
#
# The dataframe has three columns to help you out for the same, viz. `actor_1_facebook_likes`, `actor_2_facebook_likes`, and `actor_3_facebook_likes`. Your objective is to find the trios which has the most number of Facebook likes combined. That is, the sum of `actor_1_facebook_likes`, `actor_2_facebook_likes` and `actor_3_facebook_likes` should be maximum.
# Find out the top 5 popular trios, and output their names in a list.
#

# In[129]:

# Write your code here
inp['Trio']=inp['actor_1_facebook_likes']+ inp['actor_2_facebook_likes']+inp['actor_3_facebook_likes']
inp.sort_values(by='Trio',inplace=True,ascending=False,ignore_index=True)
top_5_popular_trio=inp[['Trio','Title']][0:5].values.tolist()
top_5_popular_trio

# - ### Subtask 2.5: Runtime Analysis
#
# There is a column named `Runtime` in the dataframe which primarily shows the length of the movie. It might be intersting to see how this variable this distributed. Plot a `histogram` or `distplot` of seaborn to find the `Runtime` range most of the movies fall into.

# In[130]:

# Runtime histogram/density plot
plt.figure(figsize=[8,4])
sns.distplot(inp['Runtime'])
plt.title('Movie Runtime Distrubition',fontdict={'fontsize':18,'fontweight':6})
plt.show()

# **`Checkpoint 3:`** Most of the movies appear to be sharply 2 hour-long.

# - ### Subtask 2.6: R-Rated Movies
#
# Although R rated movies are restricted movies for the under 18 age group, still there are vote counts from that age group. Among all the R rated movies that have been voted by the under-18 age group, find the top 10 movies that have the highest number of votes i.e.`CVotesU18` from the `movies` dataframe. Store these in a dataframe named `PopularR`.

# In[131]:

# Write your code here
R_rated= inp[inp['content_rating']=='R']
popular_R=R_rated.sort_values(by='CVotesU18',ascending=False,ignore_index=True)[0:10]
popular_R[0:6]

# **`Checkpoint 4:`** Are these kids watching `Deadpool` a lot?

#

# ## Task 3 : Demographic analysis
#
# If you take a look at the last columns in the dataframe, most of these are related to demographics of the voters (in the last subtask, i.e., 2.8, you made use one of these columns - CVotesU18). We also have three genre columns indicating the genres of a particular movie. We will extensively use these columns for the third and the final stage of our assignment wherein we will analyse the voters across all demographics and also see how these vary across various genres. So without further ado, let's get started with `demographic analysis`.

# -  ###  Subtask 3.1 Combine the Dataframe by Genres
#
# There are 3 columns in the dataframe - `genre_1`, `genre_2`, and `genre_3`. As a part of this subtask, you need to aggregate a few values over these 3 columns.
# 1. First create a new dataframe `df_by_genre` that contains `genre_1`, `genre_2`, and `genre_3` and all the columns related to **CVotes/Votes** from the `movies` data frame. There are 47 columns to be extracted in total.
# 2. Now, Add a column called `cnt` to the dataframe `df_by_genre` and initialize it to one. You will realise the use of this column by the end of this subtask.
# 3. First group the dataframe `df_by_genre` by `genre_1` and find the sum of all the numeric columns such as `cnt`, columns related to CVotes and Votes columns and store it in a dataframe `df_by_g1`.
# 4. Perform the same operation for `genre_2` and `genre_3` and store it dataframes `df_by_g2` and `df_by_g3` respectively.
# 5. Now that you have 3 dataframes performed by grouping over `genre_1`, `genre_2`, and `genre_3` separately, it's time to combine them. For this, add the three dataframes and store it in a new dataframe `df_add`, so that the corresponding values of Votes/CVotes get added for each genre.There is a function called `add()` in pandas which lets you do this.
# 6. The column `cnt` on aggregation has basically kept the track of the number of occurences of each genre.Subset the genres that have atleast 10 movies into a new dataframe `genre_top10` based on the `cnt` column value.
# 7. Now, take the mean of all the numeric columns by dividing them with the column value `cnt` and store it back to the same dataframe. We will be using this dataframe for further analysis in this task unless it is explicitly mentioned to use the dataframe `movies`.
# 8. Since the number of votes can't be a fraction, type cast all the CVotes related columns to integers. Also, round off all the Votes related columns upto two digits after the decimal point.
#

# In[132]:

# Create the dataframe df_by_genre
column_list=[]
for i in inp.columns:
    if i.startswith('CV') | i.startswith('V') | i.startswith('ge'):
        column_list.append(i)
df_by_genre=inp.loc[:,column_list]

# In[133]:

# Create a column cnt and initialize it to 1
df_by_genre['cnt']=1

# In[134]:

# Group the movies by individual genres
df_by_g1=df_by_genre.groupby('genre_1')
df_by_g2=df_by_genre.groupby('genre_2')
df_by_g3=df_by_genre.groupby('genre_3')

# In[135]:

# Add the grouped data frames and store it in a new data frame
df_by_g1=pd.DataFrame(df_by_g1.sum())
df_by_g2=pd.DataFrame(df_by_g2.sum())
df_by_g3=pd.DataFrame(df_by_g3.sum())
add_df1_df2=df_by_g1.add(df_by_g2, fill_value=0)
df_add=add_df1_df2.add(df_by_g3, fill_value=0)
gener_top10=df_add[df_add['cnt']>=10]
gener_top10.head()

# In[136]:

# Extract genres with atleast 10 occurences
gener_top10 = df_add[df_add["cnt"] > 10].sort_values(by="cnt",ascending=False)
gener_top10

# In[137]:

# Take the mean for every column by dividing with cnt
for i in range(0,44):
    gener_top10.iloc[:,i] = gener_top10.iloc[:,i] / gener_top10.iloc[:,-1]
gener_top10

# In[138]:

# Rounding off the columns of Votes to two decimals
gener_top10.iloc[:,27:44] = round(gener_top10.iloc[:,27:44],2)

# In[139]:

# Converting CVotes to int type
genre_top10.iloc[:,0:27] = genre_top10.iloc[:,0:27].astype(int)

# If you take a look at the final dataframe that you have gotten, you will see that you now have the complete information about all the demographic (Votes- and CVotes-related) columns across the top 10 genres. We can use this dataset to extract exciting insights about the voters!

# -  ###  Subtask 3.2: Genre Counts!
#
# Now let's derive some insights from this data frame. Make a bar chart plotting different genres vs cnt using seaborn.

# In[140]:

gener_top10['cnt'].plot.bar()
plt.show()

# **`Checkpoint 5:`** Is the bar for `Drama` the tallest?

# -  ###  Subtask 3.3: Gender and Genre
#
# If you have closely looked at the Votes- and CVotes-related columns, you might have noticed the suffixes `F` and `M` indicating Female and Male. Since we have the vote counts for both males and females, across various age groups, let's now see how the popularity of genres vary between the two genders in the dataframe.
#
# 1. Make the first heatmap to see how the average number of votes of males is varying across the genres. Use seaborn heatmap for this analysis. The X-axis should contain the four age-groups for males, i.e., `CVotesU18M`,`CVotes1829M`, `CVotes3044M`, and `CVotes45AM`. The Y-axis will have the genres and the annotation in the heatmap tell the average number of votes for that age-male group.
#
# 2. Make the second heatmap to see how the average number of votes of females is varying across the genres. Use seaborn heatmap for this analysis. The X-axis should contain the four age-groups for females, i.e., `CVotesU18F`,`CVotes1829F`, `CVotes3044F`, and `CVotes45AF`. The Y-axis will have the genres and the annotation in the heatmap tell the average number of votes for that age-female group.
#
# 3. Make sure that you plot these heatmaps side by side using `subplots` so that you can easily compare the two genders and derive insights.
#
# 4. Write your any three inferences from this plot. You can make use of the previous bar plot also here for better insights.
#
# 5. Repeat subtasks 1 to 4, but now instead of taking the CVotes-related columns, you need to do the same process for the Votes-related columns. These heatmaps will show you how the two genders have rated movies across various genres.
#
# -  Note : Use `genre_top10` dataframe for this subtask

# In[141]:

# 1st set of heat maps for CVotes-related columns
plt.figure(figsize=(20,8))
plt.subplot(1,2,1)
heatmap_m = sns.heatmap(gener_top10.iloc[:,13:23:3],annot=True,cmap="Spectral_r")
bottom, top = heatmap_m.get_ylim()
heatmap_m.set_ylim(bottom + 0.5, top - 0.5)
plt.subplot(1,2,2)
heatmap_f = sns.heatmap(gener_top10.iloc[:,14:24:3],annot=True,cmap="Spectral_r")
bottom, top = heatmap_f.get_ylim()
heatmap_f.set_ylim(bottom + 0.5, top - 0.5)
plt.show()

# In[142]:

# 2nd set of heat maps for Votes-related columns
plt.figure(figsize=(20,8))
plt.subplot(1,2,1)
heatmap_m = sns.heatmap(gener_top10.iloc[:,30:40:3],annot=True,cmap="Spectral_r")
bottom, top = heatmap_m.get_ylim()
heatmap_m.set_ylim(bottom + 0.5, top - 0.5)
plt.subplot(1,2,2)
heatmap_f = sns.heatmap(gener_top10.iloc[:,31:41:3],annot=True,cmap="Spectral_r")
bottom, top = heatmap_f.get_ylim()
heatmap_f.set_ylim(bottom + 0.5, top - 0.5)
plt.show()

# -  ###  Subtask 3.4: US vs non-US Cross Analysis
#
# The dataset contains both the US and non-US movies. Let's analyse how both the US and the non-US voters have responded to the US and the non-US movies.
#
# 1. Create a column `IFUS` in the dataframe `movies`. The column `IFUS` should contain the value "USA" if the `Country` of the movie is "USA". For all other countries other than the USA, `IFUS` should contain the value `non-USA`.
#
#
# 2. Now make a boxplot that shows how the number of votes from the US people i.e. `CVotesUS` is varying for the US and non-US movies. Make use of the column `IFUS` to make this plot. Similarly, make another subplot that shows how non US voters have voted for the US and non-US movies by plotting `CVotesnUS` for both the US and non-US movies. Write any of your two inferences/observations from these plots.
#
#
# 3. Again do a similar analysis but with the ratings. Make a boxplot that shows how the ratings from the US people i.e. `VotesUS` is varying for the US and non-US movies. Similarly, make another subplot that shows how `VotesnUS` is varying for the US and non-US movies. Write any of your two inferences/observations from these plots.
#
# Note : Use `movies` dataframe for this subtask.

# In[143]:

# Creating IFUS column
inp["IFUS"] = "USA"
inp.loc[inp["Country"] != "USA","IFUS"] = "non-USA"
inp

# In[144]:

# Box plot - 1: CVotesUS(y) vs IFUS(x)
plt.figure(figsize=(20,8))
plt.subplot(1,2,1)
sns.boxplot(x=inp["IFUS"],y=inp["CVotesUS"])
plt.subplot(1,2,2)
sns.boxplot(x=inp["IFUS"],y=inp["CVotesnUS"])
plt.show()

# In[145]:

# Box plot - 2: VotesUS(y) vs IFUS(x)
plt.figure(figsize=(20,8))
plt.subplot(1,2,1)
sns.boxplot(x=inp["IFUS"],y=inp["VotesUS"])
plt.subplot(1,2,2)
sns.boxplot(x=inp["IFUS"],y=inp["VotesnUS"])
plt.show()

# -  ###  Subtask 3.5:  Top 1000 Voters Vs Genres
#
# You might have also observed the column `CVotes1000`. This column represents the top 1000 voters on IMDb and gives the count for the number of these voters who have voted for a particular movie. Let's see how these top 1000 voters have voted across the genres.
#
# 1. Sort the dataframe genre_top10 based on the value of `CVotes1000`in a descending order.
#
# 2. Make a seaborn barplot for `genre` vs `CVotes1000`.
#
# 3. Write your inferences. You can also try to relate it with the heatmaps you did in the previous subtasks.
#
#
#

# In[146]:

# Sorting by CVotes1000
gener_top10.sort_values(by='CVotes1000',ascending=False)

# In[147]:

# Bar plot
plt.figure(figsize=(10,5))
sns.barplot(x=gener_top10.index,y=gener_top10["CVotes1000"])

# **`Checkpoint 6:`** The genre `Romance` seems to be most unpopular among the top 1000 voters.

#
#
#

# In[ ]:


# 데이터 프레임 정렬: Dataframe.sort_values()
# 튜플 정렬 : sorted(tuple,key)
# 리스트 정렬 : sort(),sorted(list)
import pandas as pd

pdf = pd.DataFrame({'seq':[1,3,2],'name':['park','lee','choi'],'age' : [30,20,40]})
print(pdf)
#    seq  name  age
# 0    1  park   30
# 1    3   lee   20
# 2    2  choi   40

#정렬 기준을 정할 수 있다.
print(pdf.sort_values(by=['seq']))                                      #seq중심으로 오름차순
#    seq  name  age
# 0    1  park   30
# 2    2  choi   40
# 1    3   lee   20

print(pdf.sort_values(by=['seq'],axis=0,ascending=False))
#    seq  name  age
# 1    3   lee   20
# 2    2  choi   40
# 0    1  park   30

print('='*50)

pdf.sort_values(by=['seq'],axis=0)
print(pdf)
#    seq  name  age
# 0    1  park   30
# 1    3   lee   20
# 2    2  choi   40

pdf.sort_values(by=['seq'],axis=0,inplace=True)                             #inplace=True : 정렬 기준을 저장(없으면 따로 저장:변수 = 변수.sort_values...)
print(pdf)
#    seq  name  age
# 0    1  park   30
# 2    2  choi   40
# 1    3   lee   20

import numpy as np
pdf = pd.DataFrame({'seq':[1,3,np.nan],'name':['park','lee','choi'],'age' : [30,20,40]})

pdf.sort_values(by=['seq'],axis=0,inplace=True,na_position='first')                             #na_position='first' : na의 정렬 기준 지정
print(pdf)
#    seq  name  age
# 2  NaN  choi   40
# 0  1.0  park   30
# 1  3.0   lee   20

pt = [(1,'park',30),(3,'lee',20),(2,'choi',40)]
print(pt)
# [(1, 'park', 30), (3, 'lee', 20), (2, 'choi', 40)]

print(sorted(pt,key=lambda ptf:ptf[0]))                                     #ptf[0] :  0번째 열에 맞춰 오름차순 정렬
# [(1, 'park', 30), (2, 'choi', 40), (3, 'lee', 20)]

print(sorted(pt,key=lambda ptf:ptf[1]))                                     #ptf[1] :  1번째 열에 맞춰 오름차순 정렬
# [(2, 'choi', 40), (3, 'lee', 20), (1, 'park', 30)]

print(sorted(pt,key=lambda ptf:ptf[2]))                                     #ptf[2] :  2번째 열에 맞춰 오름차순 정렬
# [(3, 'lee', 20), (1, 'park', 30), (2, 'choi', 40)]

print(sorted(pt,reverse=True, key=lambda ptf:ptf[2]))
# [(2, 'choi', 40), (1, 'park', 30), (3, 'lee', 20)]

#리스트 : sorted(list), sort
mlist = [9,4,1,2,7]

print(sorted(mlist))                                                     #정렬 된 결과가 저장이 따로 되진 않는다
# [1, 2, 4, 7, 9]

print(mlist)
# [9, 4, 1, 2, 7]

mlist.sort()                                                              #정렬 된 결과가 자동 저장
print(mlist)
# [1, 2, 4, 7, 9]

Seri = pd.Series([ 10.,11.,12.,13.,14.])
print(Seri)
# 0    10.0
# 1    11.0
# 2    12.0
# 3    13.0
# 4    14.0
# dtype: float64

print(Seri[3])
# 13.0

print(Seri[:3])
# 0    10.0
# 1    11.0
# 2    12.0
# dtype: float64

#평균이 12 이상인 행만 추출
print(Seri[Seri>=Seri.mean()])
# 2    12.0
# 3    13.0
# 4    14.0
# dtype: float64

print(Seri[[3,4,2]])
# 3    13.0
# 4    14.0
# 2    12.0
# dtype: float64

Seri_ix = pd.Series([10.,11.,12.,13.,14.],index=['a','b','c','d','e'])
print(Seri_ix)
# a    10.0
# b    11.0
# c    12.0
# d    13.0
# e    14.0
# dtype: float64

print(Seri_ix[['a','b','d']])
# a    10.0
# b    11.0
# d    13.0
# dtype: float64

print(Seri_ix.get(['a','b','d']))
# a    10.0
# b    11.0
# d    13.0
# dtype: float64

Seri_ix['c']=100
print(Seri_ix)
# a     10.0
# b     11.0
# c    100.0
# d     13.0
# e     14.0
# dtype: float64

#특정 인덱스가 있는지 확인
print('c' in Seri_ix)
# True

# ix, loc, iloc
# ix : 위치를 지정하여 데이터를 참조, 레이블을 이용하여 데이터를 참조
# 레이블을 이용하여 데이터 참조 시 ix속성에 수치값을 주는 경우에는 loc와 동일한 결과
# loc : 레이블을 이용하여 데이터를 참조(위치를 이용하여 참조할 수 없다.)
# iloc : 위치를 이용하여 데이터를 참조(레이블을 이용하여 참조할 수 없다.)

Seri = pd.Series(np.nan, index=[19,18,17,16,15,1,2,3,4,5])
print(Seri)
# True
# 19   NaN
# 18   NaN
# 17   NaN
# 16   NaN
# 15   NaN
# 1    NaN
# 2    NaN
# 3    NaN
# 4    NaN
# 5    NaN
# dtype: float64

print(Seri.iloc[:3])                                   #0~2번 행까지 추출
# 19   NaN
# 18   NaN
# 17   NaN
# dtype: float64

print(Seri.loc[:3])                                   #0~7번 행까지 3번이라는 인덱스가 있는 행까지 추출
# 19   NaN
# 18   NaN
# 17   NaN
# 16   NaN
# 15   NaN
# 1    NaN
# 2    NaN
# 3    NaN
# dtype: float64

print(Seri.ix[:3])                                   #0~7번 행까지 3번이라는 인덱스가 있는 행까지 추출
# 19   NaN
# 18   NaN
# 17   NaN
# 16   NaN
# 15   NaN
# 1    NaN
# 2    NaN
# 3    NaN
# dtype: float64

from pandas import DataFrame

df = DataFrame({'c1': [0,1,2,3],
                'c2': [4,5,6,7],
                'c3':[8,9,10,np.nan]},index=['r1','r2','r3','r4'])
print(df.index)
# Index(['r1', 'r2', 'r3', 'r4'], dtype='object')
print(df.columns)
# Index(['c1', 'c2', 'c3'], dtype='object'

df_r1 = DataFrame(df,index=['r1'])
print(df_r1)
#     c1  c2   c3
# r1   0   4  8.0
print(type(df_r1))
# <class 'pandas.core.frame.DataFrame'>

df_r1 = DataFrame(df,index=['r1','r3'])
print(df_r1)
#     c1  c2    c3
# r1   0   4   8.0
# r3   2   6  10.0

df_c1 = DataFrame(df,columns=['c1','c3'])
print(df_c1)
#     c1    c3
# r1   0   8.0
# r2   1   9.0
# r3   2  10.0
# r4   3   NaN

#기존의 데이터프레임으로부터 원하는 특정 부분만 추출하여 새로운 데이터프레임 생성
df_ex = DataFrame(df,index=['r3','r1'],columns=['c3','c1'])
print(df_ex)
#       c3  c1
# r3  10.0   2
# r1   8.0   0

#기존의 데이터프레임으로부터 원하는 특정 부분만 참조
print(df[['c1','c3']])
#     c1    c3
# r1   0   8.0
# r2   1   9.0
# r3   2  10.0
# r4   3   NaN

df['csum']=df['c1']+df['c2']
print(df)
#     c1  c2    c3  csum
# r1   0   4   8.0     4
# r2   1   5   9.0     6
# r3   2   6  10.0     8
# r4   3   7   NaN    10

df = df.assign(cmul=df['c1']*df['c2'])
print(df)
#     c1  c2    c3  csum  cmul
# r1   0   4   8.0     4     0
# r2   1   5   9.0     6     5
# r3   2   6  10.0     8    12
# r4   3   7   NaN    10    21

df = df.assign(cmul2=lambda  x:x.c1*x.c2)                   # lambda :  df가 x로 들어가서 뒤에 계산식 수행 수 cmul2에 값이 대입
print(df)
#     c1  c2    c3  csum  cmul  cmul2
# r1   0   4   8.0     4     0      0
# r2   1   5   9.0     6     5      5
# r3   2   6  10.0     8    12     12
# r4   3   7   NaN    10    21     21

#행이나 열 삭제: drop,del
print(df.drop(['cmul','cmul2'],axis=1))
#     c1  c2    c3  csum
# r1   0   4   8.0     4
# r2   1   5   9.0     6
# r3   2   6  10.0     8
# r4   3   7   NaN    10

print(df.drop(['r1','r3'],axis=0))
#     c1  c2   c3  csum  cmul  cmul2
# r2   1   5  9.0     6     5      5
# r4   3   7  NaN    10    21     21

print(df.drop(['r1','r3']))                                 # axis=0 기본값이라 생략 가능
#     c1  c2   c3  csum  cmul  cmul2
# r2   1   5  9.0     6     5      5
# r4   3   7  NaN    10    21     21

del df['csum']
print(df)
#     c1  c2    c3  cmul  cmul2
# r1   0   4   8.0     0      0
# r2   1   5   9.0     5      5
# r3   2   6  10.0    12     12
# r4   3   7   NaN    21     21

print(df['c1'])
# r1    0
# r2    1
# r3    2
# r4    3
# Name: c1, dtype: int64

print(df.c1)
# r1    0
# r2    1
# r3    2
# r4    3
# Name: c1, dtype: int64

print(df[0:2])
#     c1  c2   c3  cmul  cmul2
# r1   0   4  8.0     0      0
# r2   1   5  9.0     5      5

print(df['c1'][0:2])
# r1    0
# r2    1
# Name: c1, dtype: int64

print(df.c2[0:2])
# r1    4
# r2    5
# Name: c2, dtype: int64

print(df.loc['r1'])
#     c1  c2   c3  cmul  cmul2
# r1   0   4  8.0     0      0
# r2   1   5  9.0     5      5

print(df.loc[['r1','r2']])
#     c1  c2   c3  cmul  cmul2
# r1   0   4  8.0     0      0
# r2   1   5  9.0     5      5
print(df.iloc[0:2])
#     c1  c2   c3  cmul  cmul2
# r1   0   4  8.0     0      0
# r2   1   5  9.0     5      5

print(df[0:2])
#     c1  c2   c3  cmul  cmul2
# r1   0   4  8.0     0      0
# r2   1   5  9.0     5      5

print('='*50)
print(df[df['c1']<=1])
#     c1  c2   c3  cmul  cmul2
# r1   0   4  8.0     0      0
# r2   1   5  9.0     5      5

s=['c1','c2']
print(df[s])
#     c1  c2
# r1   0   4
# r2   1   5
# r3   2   6
# r4   3   7


import pandas as pd
import numpy as np

df = pd.DataFrame({"A":[1,2,3,4], "B":[5,6,7,8]})
print(df)