#Taken from the ECBM4040 assignment #2

import numpy as np
from matplotlib import pyplot as plt
import os

try:
    from scipy.ndimage.interpolation import rotate
except ModuleNotFoundError:
    os.system('pip install scipy')
    from scipy.ndimage.interpolation import rotate

class ImageGenerator(object):
    def __init__(self, x, y):
        """
        Initialize an ImageGenerator instance.
        :param x: A Numpy array of input data. It has shape (num_of_samples, height, width, channels).
        :param y: A Numpy vector of labels. It has shape (num_of_samples, ).
        """
        self.x = x
        self.y = y
        self.N = x.shape[0]
        self.is_bright = None
        self.is_horizontal_flip = None
        self.is_vertical_flip = None
        self.is_add_noise = None
        self.translated = None
        self.rotated = None
        self.flipped = None
        self.added = None
        self.bright = None
        self.x_aug = self.x.copy()
        self.y_aug = self.y.copy()
        self.N_aug = self.N

    def create_aug_data(self):
        '''
        Combine all the data to form a augmented dataset
        '''
        if self.translated:
            self.x_aug = np.vstack((self.x_aug,self.translated[0]))
            self.y_aug = np.hstack((self.y_aug,self.translated[1]))
        if self.rotated:
            self.x_aug = np.vstack((self.x_aug,self.rotated[0]))
            self.y_aug = np.hstack((self.y_aug,self.rotated[1]))
        if self.flipped:
            self.x_aug = np.vstack((self.x_aug,self.flipped[0]))
            self.y_aug = np.hstack((self.y_aug,self.flipped[1]))
        if self.added:
            self.x_aug = np.vstack((self.x_aug,self.added[0]))
            self.y_aug = np.hstack((self.y_aug,self.added[1]))
        if self.bright:
            self.x_aug = np.vstack((self.x_aug,self.bright[0]))
            self.y_aug = np.hstack((self.y_aug,self.bright[1]))

        print("Size of training data:{}".format(self.N_aug))

    def next_batch_gen(self, batch_size, shuffle=True):
        """
        A python generator function that yields a batch of data infinitely.
        :param batch_size: The number of samples to return for each batch.
        :param shuffle: If True, shuffle the entire dataset after every sample has been returned once.
                        If False, the order or data samples stays the same.
        :return: A batch of data with size (batch_size, width, height, channels).
        """
        samples = self.x_aug.shape[0]
        total_batches = samples // batch_size
        batch_count = 0
        x = self.x_aug
        y = self.y_aug
        while True:
            if (batch_count < total_batches):
                batch_count = batch_count + 1
                yield x[(batch_count-1)*batch_size:(batch_count)*batch_size,:,:,:],y[(batch_count-1)*batch_size:(batch_count)*batch_size]
            else:
                if shuffle:
                    shuffler = np.random.permutation(samples)
                    x = x[shuffler]
                    y = y[shuffler]
                batch_count = 0

    def show(self, images):
        """
        Plot the top 16 images (index 0~15) for visualization.
        :param images: images to be shown
        """

        fig = plt.figure(figsize=(10, 10))

        for i in range(16):
            ax = fig.add_subplot(4, 4, i+1)
            ax.imshow(images[i, :].reshape(28, 28), 'gray')
            ax.axis('off')

    def translate(self, shift_height, shift_width):
        """
        Translate self.x by the values given in shift.
        :param shift_height: the number of pixels to shift along height direction. Can be negative.
        :param shift_width: the number of pixels to shift along width direction. Can be negative.
        :return translated: translated dataset
        """

        translated = np.roll(self.x,shift_height,axis = 1)
        translated = np.roll(translated,shift_width,axis = 2)
        self.translated = (translated,self.y.copy())
        self.N_aug += self.N
        return translated

    def rotate(self, angle=0.0):
        """
        Rotate self.x by the angles (in degree) given.
        :param angle: Rotation angle in degrees.
        :return rotated: rotated dataset
        """

        rotated = rotate(self.x,angle,axes = (1,2),reshape = False)
        self.rotated = (rotated,self.y.copy())
        self.N_aug += self.N
        return rotated

    def flip(self, mode='h'):
        """
        Flip self.x according to the mode specified
        :param mode: 'h' or 'v' or 'hv'. 'h' means horizontal and 'v' means vertical.
        :return flipped: flipped dataset
        """
        assert mode == 'h' or 'v' or 'hv'
        if mode == 'h':
            flipped = np.flip(self.x.copy(), axis=2)
            self.is_horizontal_flip = not self.is_horizontal_flip
        elif mode == 'v':
            flipped = np.flip(self.x.copy(), axis=1)
            self.is_vertical_flip = not self.is_vertical_flip
        elif mode == 'hv':
            flipped = np.flip(np.flip(self.x.copy(), axis=0), axis=1)
            self.is_horizontal_flip = not self.is_horizontal_flip
            self.is_vertical_flip = not self.is_vertical_flip
        else:
            raise ValueError('Mode should be \'h\' or \'v\' or \'hv\'')
        print('Vertical flip: ', self.is_vertical_flip, 'Horizontal flip: ', self.is_horizontal_flip)

        self.flipped = (flipped,self.y.copy())
        self.N_aug += self.N
        return flipped

    def add_noise(self, portion, amplitude):
        """
        Add random integer noise to self.x.
        :param portion: The portion of self.x samples to inject noise. If x contains 10000 sample and portion = 0.1,
                        then 1000 samples will be noise-injected.
        :param amplitude: An integer scaling factor of the noise.
        :return added: dataset with noise added
        """

        if not self.is_add_noise:
            self.is_add_noise = True
        noise = np.random.rand(self.x.shape[1],self.x.shape[2],self.x.shape[3])*amplitude
        samples = self.x.shape[0]
        added = self.x.copy()
        added  = added[0:int(samples*portion),:,:,:] + noise
        self.added = (added,self.y.copy())
        self.N_aug += self.N
        return added

    def brightness(self, factor):
        """
        Scale the pixel values to increase the brightness
        :param factor: A number greater than or equal to 1 that decides how each pixel in the image will be scaled. If factor is 2, then
                       all pixel values will be doubled.
        :return bright: dataset with increased brightness
        """
        assert factor >= 1
        if not self.is_bright:
            self.is_bright = True
        bright = self.x.copy()
        for i in range(bright.shape[0]):
            bright[i, :, :, :] = (bright[i,:,:,:] * factor).astype(int)
            bright[i, :, :, :][bright[i,:,:,:] >= 255] = 255

        self.bright = (bright, self.y.copy())
        self.N_aug += self.N
        print("Brightness increased by a factor of:", factor)
        return bright


# Copyright (c) 2020 Huawei Technologies Co., Ltd
# Copyright (c) 2019, Facebook CORPORATION.
# All rights reserved.
#
# Licensed under the BSD 3-Clause License  (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch
import numpy as np
from common_utils import TestCase, run_tests
from common_device_type import dtypes, instantiate_device_type_tests
from util_test import create_common_tensor

class TestFloor(TestCase):
    def cpu_op_exec(self, input):
        output = torch.floor(input)
        output = output.numpy()
        return output

    def npu_op_exec(self, input):
        output = torch.floor(input)
        output = output.to("cpu")
        output = output.numpy()
        return output

    def cpu_op_inter_exec(self, input):
        torch.floor_(input)
        output = input.numpy()
        return output

    def npu_op_inter_exec(self, input):
        torch.floor_(input)
        output = input.to("cpu")
        output = output.numpy()
        return output

    def cpu_op_out_exec(self, input, output):
        torch.floor(input, out = output)
        output = output.numpy()
        return output

    def npu_op_out_exec(self, input, output):
        torch.floor(input, out = output)
        output = output.to("cpu")
        output = output.numpy()
        return output

    def test_floor_float32_shape_format(self, device):
        format_list = [0, 3]
        shape_list = [[256, 1, 1, 1], [1024, 32, 7, 7], [1024, 32, 7], [1024, 32], [1024]]
        shape_format = [
            [np.float32, i, j] for i in format_list for j in shape_list
        ]
        for item in shape_format:
            cpu_input, npu_input = create_common_tensor(item, 1, 100)
            cpu_output = self.cpu_op_exec(cpu_input)
            npu_output = self.npu_op_exec(npu_input)
            self.assertRtolEqual(cpu_output, npu_output)

    def test_floor_inter_float32_shape_format(self, device):
        format_list = [0, 3]
        shape_list = [[256, 1, 1, 1], [1024, 32, 7, 7], [1024, 32, 7], [1024, 32], [1024]]
        shape_format = [
            [np.float32, i, j] for i in format_list for j in shape_list
        ]
        for item in shape_format:
            cpu_input, npu_input = create_common_tensor(item, 1, 100)
            cpu_output = self.cpu_op_inter_exec(cpu_input)
            npu_output = self.npu_op_inter_exec(npu_input)
            self.assertRtolEqual(cpu_output, npu_output)

    def test_floor_out_float32_shape_format(self, device):
        shape_format = [
            [[np.float32, 0, [1024, 32, 7, 7]], [np.float32, 0, [1024, 32, 7, 7]]],
            [[np.float32, 0, [1024, 32, 7]], [np.float32, 0, [1024, 32]]],
            [[np.float32, 0, [1024, 32]], [np.float32, 0, [1024, 32]]],
            [[np.float32, 0, [1024]], [np.float32, 0, [1024, 1]]],
            [[np.float32, 3, [1024, 32, 7, 7]], [np.float32, 3, [1024, 32, 7, 7]]],
            [[np.float32, 3, [1024, 32, 7]], [np.float32, 3, [1024, 32]]],
            [[np.float32, 3, [1024, 32]], [np.float32, 3, [1024, 20]]],
            [[np.float32, 3, [1024]], [np.float32, 3, [1024]]],
            ]
        for item in shape_format:
            cpu_input, npu_input = create_common_tensor(item[0], 1, 100)
            cpu_output, npu_output = create_common_tensor(item[1], 1, 100)
            cpu_output = self.cpu_op_exec(cpu_input)
            npu_output = self.npu_op_out_exec(npu_input, npu_output)
            self.assertRtolEqual(cpu_output, npu_output)

    def test_floor_float16_shape_format(self, device):
        format_list = [0, 3]
        shape_list = [[256, 1, 1, 1], [1024, 32, 7, 7], [1024, 32, 7], [1024, 32], [1024]]
        shape_format = [
            [np.float16, i, j] for i in format_list for j in shape_list
        ]
        for item in shape_format:
            cpu_input, npu_input = create_common_tensor(item, 1, 100)
            if item[0] == np.float16:
                cpu_input = cpu_input.to(torch.float32)
            cpu_output = self.cpu_op_exec(cpu_input)
            npu_output = self.npu_op_exec(npu_input)
            if item[0] == np.float16:
                cpu_output = cpu_output.astype(np.float16)
            self.assertRtolEqual(cpu_output, npu_output)

    def test_floor_inter_float16_shape_format(self, device):
        format_list = [0, 3]
        shape_list = [[256, 1, 1, 1], [1024, 32, 7, 7], [1024, 32, 7], [1024, 32], [1024]]
        shape_format = [
            [np.float16, i, j] for i in format_list for j in shape_list
        ]
        for item in shape_format:
            cpu_input, npu_input = create_common_tensor(item, 1, 100)
            if item[0] == np.float16:
                cpu_input = cpu_input.to(torch.float32)
            cpu_output = self.cpu_op_inter_exec(cpu_input)
            npu_output = self.npu_op_inter_exec(npu_input)
            if item[0] == np.float16:
                cpu_output = cpu_output.astype(np.float16)
            self.assertRtolEqual(cpu_output, npu_output)

    def test_floor_out_float16_shape_format(self, device):
        shape_format = [
            [[np.float16, 0, [1024, 32, 7, 7]], [np.float16, 0, [1024, 32, 7, 7]]],
            [[np.float16, 0, [1024, 32, 7]], [np.float16, 0, [1024, 32]]],
            [[np.float16, 0, [1024, 32]], [np.float16, 0, [1024, 32]]],
            [[np.float16, 0, [1024]], [np.float16, 0, [1024, 1]]],
            [[np.float16, 3, [1024, 32, 7, 7]], [np.float16, 3, [1024, 32, 7, 7]]],
            [[np.float16, 3, [1024, 32, 7]], [np.float16, 3, [1024, 32]]],
            [[np.float16, 3, [1024, 32]], [np.float16, 3, [1024, 20]]],
            [[np.float16, 3, [1024]], [np.float16, 3, [1024]]],
            ]
        for item in shape_format:
            cpu_input, npu_input = create_common_tensor(item[0], 1, 100)
            cpu_output, npu_output = create_common_tensor(item[1], 1, 100)
            if item[0][0] == np.float16:
                cpu_input = cpu_input.to(torch.float32)
                cpu_output = cpu_output.to(torch.float32)
            cpu_output = self.cpu_op_exec(cpu_input)
            npu_output = self.npu_op_out_exec(npu_input, npu_output)
            if item[0][0] == np.float16:
                cpu_output = cpu_output.astype(np.float16)
            self.assertRtolEqual(cpu_output, npu_output)

instantiate_device_type_tests(TestFloor, globals(), except_for="cpu")
if __name__ == "__main__":
    run_tests()


# -*- coding: utf-8 -*-
from datetime import timedelta
import operator

import numpy as np
import pytest

import pandas as pd
from pandas import Series, compat
from pandas.core.indexes.period import IncompatibleFrequency
import pandas.util.testing as tm

def _permute(obj):
    return obj.take(np.random.permutation(len(obj)))

class TestSeriesFlexArithmetic(object):
    @pytest.mark.parametrize(
        'ts',
        [
            (lambda x: x, lambda x: x * 2, False),
            (lambda x: x, lambda x: x[::2], False),
            (lambda x: x, lambda x: 5, True),
            (lambda x: tm.makeFloatSeries(),
             lambda x: tm.makeFloatSeries(),
             True)
        ])
    @pytest.mark.parametrize('opname', ['add', 'sub', 'mul', 'floordiv',
                                        'truediv', 'div', 'pow'])
    def test_flex_method_equivalence(self, opname, ts):
        # check that Series.{opname} behaves like Series.__{opname}__,
        tser = tm.makeTimeSeries().rename('ts')

        series = ts[0](tser)
        other = ts[1](tser)
        check_reverse = ts[2]

        if opname == 'div' and compat.PY3:
            pytest.skip('div test only for Py3')

        op = getattr(Series, opname)

        if op == 'div':
            alt = operator.truediv
        else:
            alt = getattr(operator, opname)

        result = op(series, other)
        expected = alt(series, other)
        tm.assert_almost_equal(result, expected)
        if check_reverse:
            rop = getattr(Series, "r" + opname)
            result = rop(series, other)
            expected = alt(other, series)
            tm.assert_almost_equal(result, expected)

class TestSeriesArithmetic(object):
    # Some of these may end up in tests/arithmetic, but are not yet sorted

    def test_empty_series_add_sub(self):
        # GH#13844
        a = Series(dtype='M8[ns]')
        b = Series(dtype='m8[ns]')
        tm.assert_series_equal(a, a + b)
        tm.assert_series_equal(a, a - b)
        tm.assert_series_equal(a, b + a)
        with pytest.raises(TypeError):
            b - a

    def test_add_series_with_period_index(self):
        rng = pd.period_range('1/1/2000', '1/1/2010', freq='A')
        ts = Series(np.random.randn(len(rng)), index=rng)

        result = ts + ts[::2]
        expected = ts + ts
        expected[1::2] = np.nan
        tm.assert_series_equal(result, expected)

        result = ts + _permute(ts[::2])
        tm.assert_series_equal(result, expected)

        msg = "Input has different freq=D from PeriodIndex\\(freq=A-DEC\\)"
        with tm.assert_raises_regex(IncompatibleFrequency, msg):
            ts + ts.asfreq('D', how="end")

    def test_operators_datetimelike(self):

        # ## timedelta64 ###
        td1 = Series([timedelta(minutes=5, seconds=3)] * 3)
        td1.iloc[2] = np.nan

        # ## datetime64 ###
        dt1 = Series([pd.Timestamp('20111230'), pd.Timestamp('20120101'),
                      pd.Timestamp('20120103')])
        dt1.iloc[2] = np.nan
        dt2 = Series([pd.Timestamp('20111231'), pd.Timestamp('20120102'),
                      pd.Timestamp('20120104')])
        dt1 - dt2
        dt2 - dt1

        # ## datetime64 with timetimedelta ###
        dt1 + td1
        td1 + dt1
        dt1 - td1
        # TODO: Decide if this ought to work.
        # td1 - dt1

        # ## timetimedelta with datetime64 ###
        td1 + dt1
        dt1 + td1

# ------------------------------------------------------------------
# Comparisons

class TestSeriesFlexComparison(object):
    def test_comparison_flex_basic(self):
        left = pd.Series(np.random.randn(10))
        right = pd.Series(np.random.randn(10))

        tm.assert_series_equal(left.eq(right), left == right)
        tm.assert_series_equal(left.ne(right), left != right)
        tm.assert_series_equal(left.le(right), left < right)
        tm.assert_series_equal(left.lt(right), left <= right)
        tm.assert_series_equal(left.gt(right), left > right)
        tm.assert_series_equal(left.ge(right), left >= right)

        # axis
        for axis in [0, None, 'index']:
            tm.assert_series_equal(left.eq(right, axis=axis), left == right)
            tm.assert_series_equal(left.ne(right, axis=axis), left != right)
            tm.assert_series_equal(left.le(right, axis=axis), left < right)
            tm.assert_series_equal(left.lt(right, axis=axis), left <= right)
            tm.assert_series_equal(left.gt(right, axis=axis), left > right)
            tm.assert_series_equal(left.ge(right, axis=axis), left >= right)

        #
        msg = 'No axis named 1 for object type'
        for op in ['eq', 'ne', 'le', 'le', 'gt', 'ge']:
            with tm.assert_raises_regex(ValueError, msg):
                getattr(left, op)(right, axis=1)

class TestSeriesComparison(object):
    def test_comparison_different_length(self):
        a = Series(['a', 'b', 'c'])
        b = Series(['b', 'a'])
        with pytest.raises(ValueError):
            a < b

        a = Series([1, 2])
        b = Series([2, 3, 4])
        with pytest.raises(ValueError):
            a == b

    @pytest.mark.parametrize('opname', ['eq', 'ne', 'gt', 'lt', 'ge', 'le'])
    def test_ser_flex_cmp_return_dtypes(self, opname):
        # GH#15115
        ser = Series([1, 3, 2], index=range(3))
        const = 2

        result = getattr(ser, opname)(const).get_dtype_counts()
        tm.assert_series_equal(result, Series([1], ['bool']))

    @pytest.mark.parametrize('opname', ['eq', 'ne', 'gt', 'lt', 'ge', 'le'])
    def test_ser_flex_cmp_return_dtypes_empty(self, opname):
        # GH#15115 empty Series case
        ser = Series([1, 3, 2], index=range(3))
        empty = ser.iloc[:0]
        const = 2

        result = getattr(empty, opname)(const).get_dtype_counts()
        tm.assert_series_equal(result, Series([1], ['bool']))

    @pytest.mark.parametrize('op', [operator.eq, operator.ne,
                                    operator.le, operator.lt,
                                    operator.ge, operator.gt])
    @pytest.mark.parametrize('names', [(None, None, None),
                                       ('foo', 'bar', None),
                                       ('baz', 'baz', 'baz')])
    def test_ser_cmp_result_names(self, names, op):
        # datetime64 dtype
        dti = pd.date_range('1949-06-07 03:00:00',
                            freq='H', periods=5, name=names[0])
        ser = Series(dti).rename(names[1])
        result = op(ser, dti)
        assert result.name == names[2]

        # datetime64tz dtype
        dti = dti.tz_localize('US/Central')
        ser = Series(dti).rename(names[1])
        result = op(ser, dti)
        assert result.name == names[2]

        # timedelta64 dtype
        tdi = dti - dti.shift(1)
        ser = Series(tdi).rename(names[1])
        result = op(ser, tdi)
        assert result.name == names[2]

        # categorical
        if op in [operator.eq, operator.ne]:
            # categorical dtype comparisons raise for inequalities
            cidx = tdi.astype('category')
            ser = Series(cidx).rename(names[1])
            result = op(ser, cidx)
            assert result.name == names[2]

def test_pow_ops_object():
    # 22922
    # pow is weird with masking & 1, so testing here
    a = Series([1, np.nan, 1, np.nan], dtype=object)
    b = Series([1, np.nan, np.nan, 1], dtype=object)
    result = a ** b
    expected = Series(a.values ** b.values, dtype=object)
    tm.assert_series_equal(result, expected)

    result = b ** a
    expected = Series(b.values ** a.values, dtype=object)

    tm.assert_series_equal(result, expected)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import dask
from dask_kubernetes import KubeCluster
import numpy as np

# In[2]:

# Specify a remote deployment using a load blanacer
dask.config.set({"kubernetes.scheduler-service-type": "LoadBalancer"})

# In[3]:

cluster = KubeCluster.from_yaml('worker-spec.yaml', namespace='dask', deploy_mode='remote')

# In[4]:

cluster.adapt(minimum=1, maximum=100)

# In[5]:

# Example usage
from dask.distributed import Client
import dask.array as da

# Connect Dask to the cluster
client = Client(cluster)

# In[6]:

client.scheduler_comm.comm.handshake_info()

# In[7]:

# Create a large array and calculate the mean
array = da.ones((1000, 1000, 1000))
print(array.mean().compute())  # Should print 1.0|

# In[8]:

print(array.mean().compute())

# In[9]:

print(array.sum().compute())

# In[10]:

dir(array)

# In[11]:

np.take(array, indices=[0, 10]).sum().compute()

# In[12]:

from time import sleep

def inc(x):
    sleep(1)
    return x + 1

def add(x, y):
    sleep(1)
    return x + y

# In[13]:

get_ipython().run_cell_magic('time', '', '# This takes three seconds to run because we call each\n# function sequentially, one after the other\n\nx = inc(1)\ny = inc(2)\nz = add(x, y)')

# In[14]:

from dask import delayed

# In[15]:

get_ipython().run_cell_magic('time', '', '# This runs immediately, all it does is build a graph\n\nx = delayed(inc)(1)\ny = delayed(inc)(2)\nz = delayed(add)(x, y)')

# In[16]:

get_ipython().run_cell_magic('time', '', 'z.compute()')

# In[17]:

data = range(1,100)

# In[18]:

results = []

for x in data:
    y = delayed(inc)(x)
    results.append(y)

total = delayed(sum)(results)
print("Before computing:", total)  # Let's see what type of thing total is
result = total.compute()
print("After computing :", result)  # After it's computed

# In[19]:

total.compute()

# In[20]:

def double(x):
    sleep(1)
    return 2 * x

def is_even(x):
    return not x % 2

# In[21]:

get_ipython().run_cell_magic('time', '', 'results = []\nfor x in data:\n    if is_even(x):  # even\n        y = delayed(double)(x)\n    else:          # odd\n        y = delayed(inc)(x)\n    results.append(y)\n\ntotal = delayed(sum)(results)\ntotal.compute()')

# In[22]:

get_ipython().run_cell_magic('time', '', 'results = []\nfor x in data:\n    def compute(x):\n        if is_even(x):  # even\n            return double(x)\n        else:          # odd\n            return inc(x)\n    y = delayed(compute)(x)\n    results.append(y)\n\ntotal = delayed(sum)(results)\ntotal.compute()')

# In[23]:

total.visualize()

# In[24]:

from dask import compute

# In[25]:

import dask.bag as db
b = db.from_sequence(range(1,100))

# In[26]:

iseven = lambda x: x % 2 == 0

# In[27]:

add = lambda x, y: x + y

# In[28]:

dict(b.foldby(iseven, add))

# In[29]:

b.foldby(iseven, add)

# In[30]:

b.foldby(iseven, add).compute()

# In[31]:

grouped = b.groupby(iseven)

# In[58]:

badsum = grouped.map(lambda kv: (kv[0], sum(kv[1])))

# In[41]:

f = client.scatter(badsum)

# In[42]:

f

# In[45]:

# Note I killed the worker in between
f

# In[59]:

f = client.scatter(badsum)

# In[51]:

f

# In[49]:

dict(f.result())

# In[55]:

# introduced network failure
f.result()

# In[56]:

f

# In[34]:

from bokeh.io import output_notebook, push_notebook
from bokeh.models.sources import ColumnDataSource
from bokeh.plotting import figure, show
import numpy as np
output_notebook()

# set up plot background
N = 500
x = np.linspace(-5, 5, N)
y = np.linspace(-5, 5, N)
xx, yy = np.meshgrid(x, y)
d = (1 - xx)**2 + 2 * (yy - xx**2)**2
d = np.log(d)

p = figure(x_range=(-5, 5), y_range=(-5, 5))
p.image(image=[d], x=-5, y=-5, dw=10, dh=10, palette="Spectral11");

# In[35]:

c = client
# a simple function with interesting minima
import time

def rosenbrock(point):
    """Compute the rosenbrock function and return the point and result"""
    time.sleep(0.1)
    score = (1 - point[0])**2 + 2 * (point[1] - point[0]**2)**2
    return point, score

# In[36]:

from dask.distributed import as_completed
from random import uniform

scale = 5                  # Intial random perturbation scale
best_point = (0, 0)        # Initial guess
best_score = float('inf')  # Best score so far
startx = [uniform(-scale, scale) for _ in range(10)]
starty = [uniform(-scale, scale) for _ in range(10)]

# set up plot
source = ColumnDataSource({'x': startx, 'y': starty, 'c': ['grey'] * 10})
p.circle(source=source, x='x', y='y', color='c')
t = show(p, notebook_handle=True)

# initial 10 random points
futures = [c.submit(rosenbrock, (x, y)) for x, y in zip(startx, starty)]
iterator = as_completed(futures)

# TODO(holden): non-blocking?
for res in iterator:
    # take a completed point, is it an improvement?
    point, score = res.result()
    if score < best_score:
        best_score, best_point = score, point
        print(score, point)

    x, y = best_point
    newx, newy = (x + uniform(-scale, scale), y + uniform(-scale, scale))

    # update plot
    source.stream({'x': [newx], 'y': [newy], 'c': ['grey']}, rollover=20)
    push_notebook(document=t)

    # add new point, dynamically, to work on the cluster
    new_point = c.submit(rosenbrock, (newx, newy))
    iterator.add(new_point)  # Start tracking new task as well

    # Narrow search and consider stopping
    scale *= 0.99
    if scale < 0.001:
        break
point

# In[37]:

dir(c)

# In[38]:

help(c)

# In[39]:

c.list_datasets()

# In[60]:

client

# In[62]:

# In[63]:

f

# In[ ]:


import numpy as np

import pandas as pd
'''
@alt(表データ|データフレーム)
@alt(カラム|列)
@alt(インデックス|行)
@alt(欠損値|NaN|未入力値)
@alt(変更する|増やす|減らす)
@alt(保存する|保存する|書き込む)
@alt(抽出する|取り出す)
@alt(読み込む|読む)
@alt(読み込んで|読んで)
@alt(全ての|すべての|全)
@alt(の名前|名)

@prefix(df;データフレーム)
@prefix(ds;[データ列|カラム])
@prefix(col;[列|カラム])
@prefix(value;[文字列|日付|値])

表データを使う
表データをインポートする
'''

import seaborn as sns

n = 1

filename = 'test.csv'
filename = 'file.csv'
filename = 'file.tsv'
filename = 'file.json'
filename = 'file.json'

col, col2, col3 = 'A', 'B', 'C'
df = pd.DataFrame(data=[[1, 2, 3], [4, 5, 6]], columns=['A', 'B', 'C'])
df2 = df
ds, ds2 = df[col], df[col2]

# 設定

print(pd.__version__)
'''
Pandasのバージョンをプリントする
Pandasのバージョンを見る
'''

pd.set_option('display.max_columns', n)
'''
@alt(表示可能な|[表示できる|表示する|表示される])
[データフレームを表示するとき、|][表示可能な|][最大|]列数を変更する
[データフレームを表示するとき、|][表示可能な|]列数の最大値をnに設定する
[データフレームを表示するとき、|]{n列まで|データフレームを}表示可能なようにする
'''

pd.set_option('display.max_rows', n)
'''
[データフレームを表示するとき、|][表示可能な|][最大|]行数を変更する
[データフレームを表示するとき、|][表示可能な|][最大|]行数をnに設定する
[データフレームを表示するとき、|]{n行まで|データフレームを}表示できるようにする
'''

pd.set_option('precision', n)
'''
[データフレームを表示するとき、|]小数点以下の表示精度を設定する
[データフレームを表示するとき、|]小数点以下[の表示精度|]をn桁に設定する
[データフレームを表示するとき、|]小数点以下n桁まで表示可能なようにする
'''

pd.set_option('expand_frame_repr', False)
'''
[データフレームを表示するとき、|]折り返しをしない[|ようにする]
[データフレームを表示するとき、|]折り返しを[オフ|無効]に設定する
'''

pd.set_option('max_colwidth', n)
'''
[データフレームを表示するとき、|]カラムの最大幅をnに設定する
'''

pd.set_option('colheader_justify', 'right')
'''
[データフレームを表示するとき、|]ヘッダー行を右寄せに設定する
'''

pd.set_option('colheader_justify', 'left')
'''
[データフレームを表示するとき、|]ヘッダー行を左寄せに設定する
'''

# 読み込み系

filename = 'file.xlsx'
pd.read_excel(filename)
'''
@prefix(filename;エクセルファイル)
@alt(エクセル|エクセル[ファイル|データ])

{エクセルから|データフレームを}読み込む
{|filenameを|[データフレームとして|]}読み込む
{|filenameから|エクセルを}読み込む
'''

pd.read_excel(filename, sheet_name=n)
'''
@test(pd=missing;filename='file.xlsx';$$)
{filenameから|n番目のシートを}[データフレームとして|]読み込む
{filenameから|nという[名前の|]シートを}[データフレームとして|]読み込む
{filenameの|n番目のシートを}[データフレームとして|]読み込む
'''

pd.read_excel(filename, sheet_name=[n, n2])
'''
@test(pd=missing;filename='file.xlsx';$$)
{filenameから|複数のシートを}読み込む
{filenameから|nとn2のシートを}読み込む
'''

pd.read_excel(filename, sheet_name=None)
'''
@test(pd=missing;filename='file.xlsx';$$)
{filenameから|全てのシートを}読み込む
'''

pd.read_csv(filename, sep=',')
'''
@test(pd=missing;filename='file.csv';$$)
@prefix(filename;CSVファイル)
@alt(CSVファイル|CSV|カンマ区切りのファイル)
{CSVファイルから|データフレームを}読み込む
{filenameから|データフレームを}読み込む
{filenameを|[|データフレームとして]}読み込む
'''

pd.read_csv(filename, sep='\t')
'''
@test(pd=missing;filename='file.tsv';$$)
@prefix(filename;TSVファイル)
@alt(TSVファイル|TSV|タブ区切りのファイル)
{TSVファイルから|データフレームを}読み込む
{filenameから|データフレームを}読み込む
{filenameを|[|データフレームとして]}読み込む
'''

names = ['A', 'B']
sheet_name = names
'''
@test($$;sheet_name)
＜オプション＞[読み込む|エクセルの|]シートの名前をnamesに設定する
'''

index_col = 0
'''
@test($$;index_col)
@alt(の_|)
＜オプション＞[先頭の_|最初の]カラムをインデックスに設定する
'''

index_col = n
'''
＜オプション＞n番目のカラムをインデックスに設定する
'''

index_col = None
'''
＜オプション＞インデックスを[自動的な|]連番に設定する
＜オプション＞どのカラムもインデックスに[設定|]しない
'''

header = 0
'''
@alt(ヘッダ|カラムの名前)
＜オプション＞[先頭の|最初の]行をヘッダに設定する
'''

header = None
'''
@test($$;header)
＜オプション＞ヘッダを[自動的な|]連番に設定する
＜オプション＞どの行もヘッダに[|設定]しない
'''

header = names
'''
@test($$;header)
＜オプション＞ヘッダをnamesに設定する
'''

names = names
'''
@test($$;names)
＜オプション＞namesをカラムの名前に設定する
'''

usecols = names
'''
@test($$;usecols)
＜オプション＞読み込む行番号をnamesで指定する
'''

skiprows = names
'''
@test($$;skiprows)
＜オプション＞[読み込まない|スキップする|無視する]列番号をnamesで指定する
'''

skipfooter = n
'''
@test($$;skipfooter)
＜オプション＞[読み込まない|スキップする|無視する]フッタをnに設定する
'''

pd.read_csv(filename, header=None)
'''
@test(pd=missing;filename='file.csv';$$)
{filenameを|ヘッダ[を指定せず|なしで]}読み込む
'''

pd.read_csv(filename, index_col=n)
'''
@test(pd=missing;filename='file.csv';$$)
{CSVファイルfilenameを|n番目のカラムをインデックス[と|に]して}読み込む
文字列filenameから{CSVファイルを|n番目のカラムをインデックス[と|に]して}読み込む
'''

pd.read_csv(filename, encoding='shift_jis')
'''
@test(pd=missing;filename='file.csv';$$)
{filenameを|[SJISで|文字化けしないように]}読み込む
filenameから{CSVファイルを|[SJISで|文字化けしないように]}読み込む
'''

__X__ = 'utf-8'
pd.read_csv(filename, sep='\t', encoding=__X__)
'''
@test(pd=missing;filename='file.tsv';$$)
@X('utf-8';'shift_jis')
@Y('UTF8';シフトJIS)

{TSV[形式の|]ファイルから|データフレームを}読み込む
{filenameから|データフレームを}読み込む
{filenameを|[|データフレームとして]}読み込む
'''

pd.read_json(filename, orient='records', lines=True)
'''
@prefix(filename;JSONLファイル)

{JSONL[形式の|]ファイルから|データフレームを}読み込む
{filenameから|データフレームを}読み込む
{filenameを|[|データフレームとして]}読み込む
'''

#

# write

df.to_excel(filename)
'''
@alt(ファイル名|名前)

{dfを|filenameに}保存する
{dfを|エクセル[ファイル|形式|]で_|filenameに}保存する
'''

df.to_csv(filename)
'''
{dfを|filenameに}保存する
{dfを|CSV[ファイル|形式|]で_|filenameに}保存する
'''

df.to_csv(filename, sep='\t')
'''
{dfを|filenameに}保存する
{dfを|タブ区切りで_|filenameに}保存する
{dfを|TSV[ファイル|形式|]で_|filenameに}保存する
'''

df.to_csv(filename, header=None)
'''
@alt(ヘッダを付けずに|ヘッダなしで)
{dfを|filenameに|ヘッダを付けずに}保存する
'''

df.to_csv(filename, index=None)
'''
@alt(インデックスを付けずに|インデックスなしで)
{dfを|filenameに|インデックスを付けずに}保存する
'''

df.to_csv(filename, encoding='utf_8_sig')
'''
@test(df=missing;filename='file.txt';$$)
@alt(BOM付きで|BOMを付けて|[Windowsで|]文字化けしないように)
{dfを|filenameに|BOM付きで}保存する
'''

df.to_csv(filename, encoding='shift_jis')
'''
@test(df=missing;filename='file.txt';$$)
{dfを|filenameに|SJISで}保存する
'''

df.to_csv(filename, float_format='%.3f')
'''
@test(pd=missing;filename='file.txt';$$)
保存するCSVファイルの小数点以下の桁数を設定する
{dfを|filenameに|小数点以下3桁まで}保存する
'''


"""
==========================================
Reprojecting Images to Different Observers
==========================================

This example demonstrates how you can reproject images to the view from
different observers, we use both AIA and STEREO A data to demonstrate this.

You will need `reproject <https://reproject.readthedocs.io/en/stable/>`__ v0.6 or higher installed.
"""
# sphinx_gallery_thumbnail_number = 2

import matplotlib.pyplot as plt
from reproject import reproject_interp

import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.wcs import WCS

import sunpy.map
from sunpy.coordinates import get_body_heliographic_stonyhurst
from sunpy.net import Fido
from sunpy.net import attrs as a

######################################################################
# In this example we are going to make a lot of side by side figures, so
# let's change the default figure size.

plt.rcParams['figure.figsize'] = (16, 8)

######################################################################
# Let’s download an EUV image from both AIA and STEREO A, when their
# separation was around 90 degrees.

stereo = (a.vso.Source('STEREO_A') &
          a.Instrument('EUVI') &
          a.Time('2010-08-19', '2010-08-19T00:10:00'))
aia = (a.Instrument('AIA') &
       a.vso.Sample(24 * u.hour) &
       a.Time('2010-08-19', '2010-08-19T00:10:00'))
wave = a.Wavelength(17 * u.nm, 18 * u.nm)

res = Fido.search(wave, aia | stereo)
files = Fido.fetch(res)

######################################################################
# Create a map for each image.

map_aia, map_stereo = sunpy.map.Map(sorted(files))

# We downsample these maps to reduce memory consumption, you can comment this
# out.
map_aia = map_aia.resample((512, 512) * u.pix)
map_stereo = map_stereo.resample((512, 512) * u.pix)

fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1, projection=map_aia)
map_aia.plot(axes=ax1)
ax2 = fig.add_subplot(1, 2, 2, projection=map_stereo)
map_stereo.plot(axes=ax2)

######################################################################
# We now need to construct an output WCS. Because we want to downsample
# the image (for performance) we build a custom header using
# ``sunpy.map.make_fitswcs_header`` but we use a lot of the ``map_aia``
# properties to do it. We use the reference coordinate from the AIA image,
# and make the scale 4x larger to compensate for the fact we are making
# the resolution 4x lower.

out_shape = (512, 512)
out_header = sunpy.map.make_fitswcs_header(
    out_shape,
    map_aia.reference_coordinate,
    scale=u.Quantity(map_aia.scale)*4,
    instrument="EUVI",
    observatory="AIA Observer",
    wavelength=map_stereo.wavelength
)

######################################################################
# Next we construct an `~astropy.wcs.WCS` object from the header.
# Currently `~astropy.wcs.WCS` does not understand the observer
# position, so we manually set that.

out_wcs = WCS(out_header)
out_wcs.heliographic_observer = map_aia.reference_coordinate.observer

######################################################################
# We can now reproject the STEREO map to this output `~astropy.wcs.WCS`.
# Here we are using the fastest but least accurate method of reprojection,
# `reproject.reproject_interp`, a more accurate but slower method is
# `reproject.reproject_adaptive`.

output, footprint = reproject_interp(map_stereo, out_wcs, out_shape)

######################################################################
# We can now plot the STEREO image as seen from the position of SDO, next
# to the AIA image.

outmap = sunpy.map.Map(output, out_header)
outmap.plot_settings = map_stereo.plot_settings

fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1, projection=map_aia)
map_aia.plot(axes=ax1)
ax2 = fig.add_subplot(1, 2, 2, projection=outmap)
outmap.plot(axes=ax2)

######################################################################
# AIA as Seen from Mars
# =====================
#
# We can also change the observer of the AIA image to any observer
# coordinate. SunPy provides a function which can get the observer
# coordinate for any known body. In this example we are going to use Mars.

mars = get_body_heliographic_stonyhurst('mars', map_aia.date)

######################################################################
# We now generate a target WCS, to do this we need to generate a reference
# coordinate, which is similar to the aia frame, but scaled down by 4x and
# with the observer at Mars. To do this we generate a new reference
# coordinate.

mars_ref_coord = SkyCoord(map_aia.reference_coordinate.Tx,
                          map_aia.reference_coordinate.Ty,
                          obstime=map_aia.reference_coordinate.obstime,
                          observer=mars,
                          frame="helioprojective")

######################################################################
# then a header

out_shape = (512, 512)
mars_header = sunpy.map.make_fitswcs_header(
    out_shape,
    mars_ref_coord,
    scale=u.Quantity(map_aia.scale)*4,
    rotation_matrix=map_aia.rotation_matrix,
    instrument="AIA",
    wavelength=map_aia.wavelength
)

######################################################################
# Once again we need to generate a `~astropy.wcs.WCS` and then manually
# set the observer location.

mars_wcs = WCS(mars_header)
mars_wcs.heliographic_observer = mars

output, footprint = reproject_interp(map_aia, mars_wcs, out_shape)

######################################################################
# We generate the output map and plot it next to the original image.

outmap = sunpy.map.Map((output, mars_header))
outmap.plot_settings = map_aia.plot_settings

fig = plt.figure()

ax1 = fig.add_subplot(1, 2, 1, projection=map_aia)
map_aia.plot(axes=ax1)
outmap.draw_grid(color='w')

ax2 = fig.add_subplot(1, 2, 2, projection=outmap)
outmap.plot(axes=ax2)
outmap.draw_grid(color='w')

plt.show()


#!/usr/bin/env python
# coding: utf-8

# # WeatherPy
# ----
#
# #### Note
# * Instructions have been included for each segment. You do not have to follow them exactly, but they are included to help you think through the steps.

# In[1]:

# Dependencies and Setup
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import requests
import time
from scipy.stats import linregress

# Import API key
from api_keys import weather_api_key

# Incorporated citipy to determine city based on latitude and longitude
from citipy import citipy

# Output File (CSV)
output_data_file = "output_data/cities.csv"

# Range of latitudes and longitudes
lat_range = (-90, 90)
lng_range = (-180, 180)

# ## Generate Cities List

# In[23]:

# List for holding lat_lngs and cities
lat_lngs = []
cities = []

# Create a set of random lat and lng combinations
lats = np.random.uniform(lat_range[0], lat_range[1], size=1500)
lngs = np.random.uniform(lng_range[0], lng_range[1], size=1500)
lat_lngs = zip(lats, lngs)

# Identify nearest city for each lat, lng combination
for lat_lng in lat_lngs:
    city = citipy.nearest_city(lat_lng[0], lat_lng[1]).city_name

    # If the city is unique, then add it to a our cities list
    if city not in cities:
        cities.append(city)

# Print the city count to confirm sufficient count
len(cities)

# ### Perform API Calls
# * Perform a weather check on each city using a series of successive API calls.
# * Include a print log of each city as it'sbeing processed (with the city number and city name).
#

# In[24]:

city_name_list = []
cloudiness_list = []
country_list = []
date_list = []
humidity_list = []
lat_list = []
lng_list = []
max_temp_list = []
wind_speed_list = []
index_counter = 0
set_counter = 1

base_url = "http://api.openweathermap.org/data/2.5/weather?"
units = "imperial"
query_url = f"{base_url}appid={weather_api_key}&units={units}&q="

for index, city in enumerate(cities, start = 1):
    try:
        response = requests.get(query_url + city).json()
        city_name_list.append(response["name"])
        cloudiness_list.append(response["clouds"]["all"])
        country_list.append(response["sys"]["country"])
        date_list.append(response["dt"])
        humidity_list.append(response["main"]["humidity"])
        lat_list.append(response["coord"]["lat"])
        lng_list.append(response["coord"]["lon"])
        max_temp_list.append(response['main']['temp_max'])
        wind_speed_list.append(response["wind"]["speed"])
        if index_counter > 49:
            index_counter = 0
            set_counter = set_counter + 1

        else:
            index_counter = index_counter + 1

        print(f"Processed record {index_counter} of set {set_counter} : {city}")

    except(KeyError, IndexError):
        print("City not found. Bypassing......")

# ### Convert Raw Data to DataFrame
# * Export the city data into a .csv.
# * Display the DataFrame

# In[6]:

weather_dict = pd.DataFrame({
                "City" : city_name_list,
                "Cloudiness" : cloudiness_list,
                "Country" : country_list,
                "Date" : date_list,
                "Humidity" : humidity_list,
                "Lat" : lat_list,
                "Lng" : lng_list,
                "Max Temp" : max_temp_list,
                "Wind Speed" : wind_speed_list
})

# In[7]:

weather_dict.count()

weather_dict

# ## Inspect the data and remove the cities where the humidity > 100%.
# ----
# Skip this step if there are no cities that have humidity > 100%.

# In[ ]:

# In[8]:

#  Get the indices of cities that have humidity over 100%.

# In[9]:

# Make a new DataFrame equal to the city data to drop all humidity outliers by index.
# Passing "inplace=False" will make a copy of the city_data DataFrame, which we call "clean_city_data".

# In[ ]:

# ## Plotting the Data
# * Use proper labeling of the plots using plot titles (including date of analysis) and axes labels.
# * Save the plotted figures as .pngs.

# ## Latitude vs. Temperature Plot

# In[10]:

plt.scatter(weather_dict["Lat"], weather_dict["Max Temp"], facecolor = "steelblue", edgecolor = "black")

plt.title("City Latitude vs. Max Temperature (01/17/20)")
plt.xlabel("Laitude")
plt.ylabel("Max Temperature (F)")

plt.grid(linestyle='-', linewidth=1, alpha = 0.5)

plt.savefig("../Images/City Latitude vs Max Temperature.png")

# ## Latitude vs. Humidity Plot

# In[11]:

plt.scatter(weather_dict["Lat"], weather_dict["Humidity"], facecolor = "steelblue", edgecolor = "black")
plt.title("City Latitude vs. Humidity (01/17/20)")
plt.xlabel("Laitude")
plt.ylabel("Humidity (%)")
plt.grid(linestyle='-', linewidth=1, alpha = 0.5)
plt.savefig("../Images/City Latitude vs Humidity.png")

# ## Latitude vs. Cloudiness Plot

# In[12]:

plt.scatter(weather_dict["Lat"], weather_dict["Cloudiness"], facecolor = "steelblue", edgecolor = "black")

plt.title("City Latitude vs. Cloudiness (01/17/20)")

plt.xlabel("Laitude")

plt.ylabel("Cloudiness (%)")

plt.ylim(-5,105)

plt.grid(linestyle='-', linewidth=1, alpha = 0.5)

plt.savefig("../Images/City Latitude vs Cloudiness.png")

# ## Latitude vs. Wind Speed Plot

# In[13]:

plt.scatter(weather_dict["Lat"], weather_dict["Wind Speed"], facecolor = "steelblue", edgecolor = "black")
plt.title("City Latitude vs. Wind Speed (mph) (01/17/20)")

plt.xlabel("Laitude")

plt.ylabel("Wind Speed (%)")

plt.ylim(-2,50)

plt.grid(linestyle='-', linewidth=1, alpha = 0.5)

plt.savefig("../Images/City Latitude vs Wind Speed (mph).png")

# ## Linear Regression

# In[14]:

northern_hemisphere = weather_dict.loc[weather_dict["Lat"] >= 0]
southern_hemisphere = weather_dict.loc[weather_dict["Lat"] < 0]

def linear_agression(x,y):
    print(f"The r-squared is : {round(st.pearsonr(x, y)[0],2)}")
    (slope, intercept, rvalue, pvalue, stderr) = linregress(x, y)
    regress_values = x * slope + intercept
    line_eq = "y = " + str(round(slope,2)) + "x + " + str(round(intercept,2))
    plt.scatter(x, y)
    plt.plot(x,regress_values,"r-")
    return line_eq

def annotate(line_eq, a, b):
    plt.annotate(line_eq,(a,b),fontsize=15,color="red")

# ####  Northern Hemisphere - Max Temp vs. Latitude Linear Regression

# In[25]:

equation = linear_agression(northern_hemisphere["Lat"], northern_hemisphere["Max Temp"])

annotate(equation, 0, 0)

plt.title("Northern Hemisphere - Max Temp vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Max Temp (F)")

# ####  Southern Hemisphere - Max Temp vs. Latitude Linear Regression

# In[ ]:

equation = linear_agression(southern_hemisphere["Lat"],southern_hemisphere["Max Temp"])

annotate(equation, -30, 50)

plt.title("Southern Hemisphere - Max Temp vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Max Temp (F)")

# ####  Northern Hemisphere - Humidity (%) vs. Latitude Linear Regression

# In[ ]:

equation = linear_agression(northern_hemisphere["Lat"], northern_hemisphere["Humidity"])

annotate(equation, 40, 15)

plt.title("Northern Hemisphere - Humidity (%) vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Humidity (%)")

# ####  Southern Hemisphere - Humidity (%) vs. Latitude Linear Regression

# In[ ]:

equation = linear_agression(southern_hemisphere["Lat"], southern_hemisphere["Humidity"])

annotate(equation, -40, 50)

plt.title("Southern Hemisphere - Humidity (%) vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Humidity (%)")

# ####  Northern Hemisphere - Cloudiness (%) vs. Latitude Linear Regression

# In[ ]:

equation = linear_agression(northern_hemisphere["Lat"], northern_hemisphere["Cloudiness"])

annotate(equation, 30, 40)

plt.title("Northern Hemisphere - Cloudiness (%) vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Cloudiness (%)")

# ####  Southern Hemisphere - Cloudiness (%) vs. Latitude Linear Regression

# In[ ]:

equation = linear_agression(southern_hemisphere["Lat"], southern_hemisphere["Cloudiness"])

annotate(equation, -30, 40)

plt.title("Southern Hemisphere - Cloudiness (%) vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Cloudiness (%)")

# ####  Northern Hemisphere - Wind Speed (mph) vs. Latitude Linear Regression

# In[ ]:

equation = linear_agression(northern_hemisphere["Lat"], northern_hemisphere["Wind Speed"])

annotate(equation, 40, 20)

plt.title("Northern Hemisphere - Wind Speed (mph) vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Wind Speed (mph)")

# ####  Southern Hemisphere - Wind Speed (mph) vs. Latitude Linear Regression

# In[32]:

equation = linear_agression(southern_hemisphere["Lat"], southern_hemisphere["Wind Speed"])

annotate(equation, -30, 15)

plt.title("Southern Hemisphere - Wind Speed (mph) vs. Latitude Linear Regression")

plt.xlabel("Latitude")

plt.ylabel("Wind Speed (mph)")

# In[ ]:

plt.savefig("../Images/Southern Hemisphere - Cloudiness (%) vs. Latitude Linear Regression.png")
plt.savefig("../Images/Southern Hemisphere - Humidity (%) vs. Latitude Linear Regression.png")

# In[ ]:


#!/usr/bin/env python
# coding: utf-8

#
# # 作业四：撰写项目README并完成开源
#
# ## 评分标准
# 1.格式规范（有至少3个小标题，内容完整），一个小标题5分，最高20分
#
# 2.图文并茂，一张图5分，最高20分
#
# 3.有可运行的代码，且代码内有详细注释，20分
#
# 4.代码开源到github，15分
#
# 5.代码同步到gitee，5分
#
# ## 作业目的
# 使用MarkDown撰写项目并且学会使用开源工具。
#
#
#
# ## 参考资料：
# - [如何写好一篇高质量的精选项目？](https://aistudio.baidu.com/aistudio/projectdetail/2175889)

#
# * # 基于paddlex钢材表面缺陷检测
#
# ![](https://ai-studio-static-online.cdn.bcebos.com/bc09053d79f6454ab78e1823063ca2a04be7fb45d1c64eecba1d42d18db2cb93)
# ![](https://ai-studio-static-online.cdn.bcebos.com/b37f7b1e531b4f6fb1aa685fffca96345bda5f6d30704f8f85d5cdd417616da8)
#
#
#
#

#  ## 一、项目背景介绍
# 钢铁表面质量的好坏，直接影响后续产品的制造质量。但在生产过程中，不可避免地会产生一些缺
# 陷。这些缺陷会严重影响产品质量，给企业造成经济损失。因此钢铁进行缺陷检测极为重要。传统的基于机器视觉的表面缺陷检测方法，往往采用常规图像处理算法或人工设计特征加分类器方式。这种方式成本高不够精确，在有充分的数据和高性能硬件支持的条件下，基于深度学习的表面缺陷检测可以大大提高检测精度以及检测效率。今天的项目是钢铁表面的缺陷检测，以下是[工业场景表面缺陷检测数据集及论文集](https://github.com/Charmve/Surface-Defect-Detection/blob/master/ReadmeChinese.md#1%E9%92%A2%E6%9D%90%E8%A1%A8%E9%9D%A2%E6%95%B0%E6%8D%AE%E9%9B%86neu-cls)可能对你有帮助，建议收藏。

# ## 二、数据介绍
#
# **数据集介绍**
#
# 地址：http://faculty.neu.edu.cn/yunhyan/NEU_surface_defect_database.html
#
# ![](https://ai-studio-static-online.cdn.bcebos.com/04af07309c4e4dafa5f2e6c60739c6dae1124ddaa3fa4f88aa9c7507894f79be)
#
#
# 由东北大学（NEU）发布的表面缺陷数据库，收集了热轧钢带的六种典型表面缺陷，即轧制氧化皮（RS），斑块（Pa），开裂（Cr），点蚀表面（ PS），内含物（In）和划痕（Sc）。该数据库包括1,800个200×200像素的灰度图像：六种不同类型的典型表面缺陷，每一类缺陷包含300个样本。对于缺陷检测任务，数据集提供了注释，指示每个图像中缺陷的类别和位置。对于每个缺陷，黄色框是指示其位置的边框，绿色标签是类别分数。
#
#

# In[2]:

# 解压数据集到MyDataset文件夹中
get_ipython().system('unzip data/data102850/NEU-DET.zip -d ./MyDataset/')

# ## 三、模型介绍
# PaddleX目前提供了FasterRCNN和YOLOv3两种检测结构，多种backbone模型，可满足开发者不同场景和性能的需求。本项目中采用YOLOv3-MobileNetV3作为检测模型进行钢材缺陷检测。模型优点是模型小，移动端上预测速度有优势。因为之后要部署到移动端所以我选择了这个模型。
#

# ## 四、模型训练
# PaddleX提供了丰富的视觉模型，通过查阅[PaddleX模型库](https://paddlex.readthedocs.io/zh_CN/release-1.3/appendix/model_zoo.html)，在目标检测中提供了RCNN和YOLO系列模型。在本项目中采用YOLOv3-MobileNetV3作为检测模型进行钢材缺陷检测。

# In[ ]:

import paddlex as pdx
from paddlex import transforms as T

# In[ ]:

# 定义训练和验证时的transforms
# API说明：https://github.com/PaddlePaddle/PaddleX/blob/release/2.0-rc/paddlex/cv/transforms/operators.py
train_transforms = T.Compose([
    T.MixupImage(mixup_epoch=250), T.RandomDistort(),
    T.RandomExpand(im_padding_value=[123.675, 116.28, 103.53]), T.RandomCrop(),
    T.RandomHorizontalFlip(), T.BatchRandomResize(
        target_sizes=[320, 352, 384, 416, 448, 480, 512, 544, 576, 608],
        interp='RANDOM'), T.Normalize(
            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])

eval_transforms = T.Compose([
    T.Resize(
        608, interp='CUBIC'), T.Normalize(
            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])

# In[ ]:

# 定义训练和验证所用的数据集
# API说明：https://github.com/PaddlePaddle/PaddleX/blob/release/2.0-rc/paddlex/cv/datasets/voc.py#L29
train_dataset = pdx.datasets.VOCDetection(
    data_dir='MyDataset',
    file_list='MyDataset/train_list.txt',
    label_list='MyDataset/labels.txt',
    transforms=train_transforms,
    shuffle=True)

eval_dataset = pdx.datasets.VOCDetection(
    data_dir='MyDataset',
    file_list='MyDataset/val_list.txt',
    label_list='MyDataset/labels.txt',
    transforms=eval_transforms,
    shuffle=False)

# In[ ]:

# 初始化模型，并进行训练
# 可使用VisualDL查看训练指标，参考https://github.com/PaddlePaddle/PaddleX/tree/release/2.0-rc/tutorials/train#visualdl可视化训练指标
num_classes = len(train_dataset.labels)
model = pdx.models.YOLOv3(num_classes=num_classes, backbone='MobileNetV3_ssld')

# In[ ]:

# API说明：https://github.com/PaddlePaddle/PaddleX/blob/release/2.0-rc/paddlex/cv/models/detector.py#L155
# 各参数介绍与调整说明：https://paddlex.readthedocs.io/zh_CN/develop/appendix/parameters.html
model.train(
    num_epochs=300,
    train_dataset=train_dataset,
    train_batch_size=2,
    eval_dataset=eval_dataset,
    learning_rate=0.001 / 8,
    warmup_steps=1000,
    warmup_start_lr=0.0,
    save_interval_epochs=20,
    lr_decay_epochs=[216, 243, 275],
    save_dir='output/yolov3_mobilenet')

# ## 五、模型评估
# 该部分主要是对训练好的模型进行评估，可以是用验证集进行评估，或者是直接预测结果。评估结果和预测结果尽量展示出来，增加吸引力。

# In[ ]:

import glob
import numpy as np
import threading
import time
import random
import os
import base64
import cv2
import json
import paddlex as pdx

image_name = 'MyDataset/JPEGImages/pitted_surface_174.jpg'

model = pdx.load_model('output/yolov3_mobilenet/best_model')

img = cv2.imread(image_name)
result = model.predict(img)

keep_results = []
areas = []
f = open('output/yolov3_mobilenet/result.txt','a')
count = 0
for dt in np.array(result):
    cname, bbox, score = dt['category'], dt['bbox'], dt['score']
    if score < 0.5:
        continue
    keep_results.append(dt)
    count+=1
    f.write(str(dt)+'\n')
    f.write('\n')
    areas.append(bbox[2] * bbox[3])
areas = np.asarray(areas)
sorted_idxs = np.argsort(-areas).tolist()
keep_results = [keep_results[k]
                for k in sorted_idxs] if len(keep_results) > 0 else []
print(keep_results)
print(count)
f.write("the total number is :"+str(int(count)))
f.close()

# In[ ]:

pdx.visualize_detection(image_name, result, threshold=0.5, save_dir='./output/yolov3_mobilenet')

# ## 六、总结与升华
# 本文参考作者https://aistudio.baidu.com/aistudio/personalcenter/thirdview/791590 项目来写的，通过本次项目编写，学习了编写项目的流程以及BML使用方法。本人刚开始入门机器学习，还有很多需要学习的地方。感谢创造营的老师，希望以后自己能够独立完成模型的训练与部署。![](https://ai-studio-static-online.cdn.bcebos.com/8f411a2beffb408c9f56054dffa2b7ee6945a414c01a484a97ada7c0172d916b)
#

# ## 提交链接
# aistudio链接：我在AI Studio上获得白银等级，点亮2个徽章，来互关呀~ https://aistudio.baidu.com/aistudio/usercenter
#
# github链接：
#
# gitee链接：


import pandas as pd
import numpy as np
import pytest

from features_creator.features_creator import *

@pytest.fixture
def data_df():
    data = {
        "week_payment1": [1.0, 2, 3],
        "week_payment2": [4, 5.0, 6],
        "week_payment3": [7, 8, 9.0],
        "othercolumn": [1, 1, 1]}
    df = pd.DataFrame(data)
    return df

def test_calculate_average(data_df):

    # Test for TypeError
    with pytest.raises(TypeError):
        # check if TypeError is raised when data is not a pandas dataframe
        calculate_average(1, "week_payment")
        # check if TypeError is raised when pattern is not a string
        calculate_average(data_df, 1)

    # Test for ValueError
    with pytest.raises(ValueError):
        # check if ValueError is raised when pattern is not a string
        calculate_average(data_df, "")
        # check if ValueError is raised when columns not found
        calculate_average(data_df, "not_a_column")

    # Test for return type
    assert isinstance(calculate_average(data_df, "week_payment"), np.ndarray)

    # Test the function with a dataframe with only one column
    data_1col = {
        "week_payment1": [1, 2, 3]}
    df_1col = pd.DataFrame(data_1col)
    assert np.array_equal(calculate_average(
        df_1col, "week_payment"), np.array([1, 2, 3]))

    # Test the function return correct value when there is only one row
    data_1row = {
        "week_payment1": [1]}
    df_1row = pd.DataFrame(data_1row)
    assert np.array_equal(calculate_average(
        df_1row, "week_payment"), np.array([1]))

    # Test the function return correct value
    assert np.array_equal(calculate_average(
        data_df, "week_payment"), np.array([4, 5, 6]))

def test_get_matching_column_names():
    """
    Tests the `get_matching_column_names` function.
    Verifies that it raises the correct exceptions, works in
    "normal" usage, and does not return any extra columns.
    """
    test_df = pd.DataFrame({
        "week_payment1": [1, 2, 3],
        "week_payment2": [1, 2, 3],
        "week_payment3": [1, 2, 3],
        "othercolumn": [5, 6, 7],
        "week_payment_string4": [5, 6, 7]
    })

    # Returns the correct type
    assert isinstance(get_matching_column_names(
        test_df, "week_payment"), list), "Returned the wrong data type"

    # Does not return extra columns
    assert "othercolumn" not in get_matching_column_names(
        test_df, "week_payment"), "`othercolumn` was returned"
    assert "week_payment_string7" not in get_matching_column_names(
        test_df, "week_payment"), "`week_payment_string7` was returned"

    # Raises exceptions for wrong types
    with pytest.raises(TypeError):
        get_matching_column_names("FakeDF", "week_payment")
        get_matching_column_names(test_df, [12, 34])

    # Raises an exception for no matches
    with pytest.raises(ValueError):
        get_matching_column_names(test_df, "fake_string")

    # Normal usage test
    assert get_matching_column_names(test_df, "week_payment") == [
        "week_payment1", "week_payment2", "week_payment3"], "Incorrect columns were returned"

test_df = pd.DataFrame({
        "week_payment1": [1.0, 2, 3],
        "week_payment2": [4, 5.0, 6],
        "week_payment3": [7, 8, 9.0],
        "othercolumn": [1, 1, 1]
})

@pytest.mark.parametrize(
    'json',
    (
        ### Check data type of the first argument:data
        {"data": 3.141, "pattern": "week_payment", "check": "TypeErrorInput1"},
        {"data": "test.txt", "pattern": "week_payment", "check": "TypeErrorInput1"},
        {"data": ["list", "of", "words"], "pattern": "week_payment", "check": "TypeErrorInput1"},
        ### Check data type of the second argument:pattern
        {"data": test_df, "pattern": 3.14, "check": "TypeErrorInput2"},
        {"data": test_df, "pattern": pd.DataFrame([]), "check": "TypeErrorInput2"},
        {"data": test_df, "pattern": ["list", "of", "words"], "check": "TypeErrorInput2"},
        ### Check if the dataframe has non-numeric values
        {"data": test_df.astype(str), "pattern": "week_payment", "check": "TypeErrorNumeric"},

        ### Check data type of the output
        {"data": test_df, "pattern": "week_payment", "check": "TypeErrorOutput"},
        ### Check accuracy of the output
        {"data": test_df, "pattern": "week_payment", "check": "AccuracyOutput"}
    )
)
def test_calculate_standard_deviation(json):

    """Check TypeError raised when input is not data frame."""

    if json["check"] == "TypeErrorInput1":
        with pytest.raises(TypeError):
            print(json["data"], json["pattern"])
            calculate_standard_deviation(json["data"], json["pattern"])

    if json["check"] == "TypeErrorInput2":
        with pytest.raises(TypeError):
            print(type(json["data"]), json["pattern"])
            calculate_standard_deviation(json["data"], json["pattern"])

    if json["check"] == "TypeErrorNumeric":
        with pytest.raises(TypeError):
            print(type(json["data"]), json["pattern"])
            calculate_standard_deviation(json["data"], json["pattern"])

    if json["check"] == "TypeErrorOutput":
        assert isinstance(calculate_standard_deviation(json["data"], json["pattern"]), np.ndarray), \
            "Returned the wrong data type, output should be numpy array"

    if json["check"] == "AccuracyOutput":
        # Test if the function return correct value when there is only one element
        assert calculate_standard_deviation(test_df.iloc[0:1, 0:1], "week_payment") == np.array([0]), \
            "Should return [0], if input data frame has only one element"
        # Test if the function return correct value when there is only one column
        assert np.array_equal(calculate_standard_deviation(test_df.iloc[:, 0:1], "week_payment"), np.array([0]*test_df.shape[0])), \
            "Should a column of 0, if input data frame has only one column"
        # Test if the function return correct value
        print(calculate_standard_deviation(test_df, "week_payment"))
        assert np.array_equal(calculate_standard_deviation(test_df, "week_payment"), np.array([6**0.5, 6**0.5, 6**0.5])), \
            "The result is not right"

def test_calculate_percentage_change():
    """Test calculate_percentage_change function"""
    # Create dataset
    test_df = pd.DataFrame(
        {
            "subscriber_id": [1, 2, 3],
            "data_usage1": [5, 10, 15],
            "data_usage2": [20, 10, 10],
            "data_usage3": [25, 10, 5],
            "data_usage4": [15, 20, 25],
        }
    )

    # Raises errors for wrong input type
    with pytest.raises(TypeError):
        # Check df
        calculate_percentage_change([1, 2, 3, 4], "data_usage")

    with pytest.raises(TypeError):
        # Check pattern
        calculate_percentage_change(test_df, ["data_usage"])

    with pytest.raises(TypeError):
        # Check compare_period
        calculate_percentage_change(test_df, "data_usage", compare_period="1, 1")

    with pytest.raises(TypeError):
        # Check time_filter
        calculate_percentage_change(
            test_df, "data_usage", compare_period=(1, 1), time_filter="1, 3"
        )

    # Check Value error
    with pytest.raises(ValueError):
        calculate_percentage_change(test_df, "data_usage", compare_period=(1, 4))

    with pytest.raises(ValueError):
        calculate_percentage_change(
            test_df, "data_usage", compare_period=(1, 1), time_filter=(1, 5)
        )

    # Check return type
    assert isinstance(calculate_percentage_change(test_df, "data_usage"), np.ndarray)

    assert isinstance(
        calculate_percentage_change(
            test_df, "data_usage", compare_period=(1, 1), time_filter=(1, 3)
        ),
        np.ndarray,
    )

    # Check percentage_change return values
    # Value for comparison are calculated manually using a online calculator
    np.testing.assert_allclose(
        calculate_percentage_change(test_df, "data_usage"),
        np.array([-37.5, -33.33333333, -16.66666667]),
    )
    np.testing.assert_allclose(
        calculate_percentage_change(test_df, "data_usage", compare_period=(1, 2)),
        np.array([-77.77777778, 0.0, 100.0]),
    )
    np.testing.assert_allclose(
        calculate_percentage_change(test_df, "data_usage", compare_period=(3, 1)),
        np.array([11.11111111, -50.0, -60.0]),
    )
    np.testing.assert_allclose(
        calculate_percentage_change(
            test_df, "data_usage", compare_period=(1, 2), time_filter=(1, 3, 4)
        ),
        np.array([-75.0, -33.33333333, 0.0]),
    )
    np.testing.assert_allclose(
        calculate_percentage_change(
            test_df, "data_usage", compare_period=(1, 2), time_filter=(4, 2, 1)
        ),
        np.array([-71.42857143, -33.33333333, -14.28571429]),
    )