from scipy import spatial
import os
import pickle
from six.moves import cPickle
from keras import backend as K
from keras.layers import Input, Conv2D, MaxPooling2D
from keras.models import Model
import cv2
from matplotlib import pyplot as plt
import numpy as np
import operator

from flask import Flask, request, jsonify
import numpy as np
from PIL import Image

app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello World!'

@app.route('/test', methods=["POST"])
def test():
    data = {}
    if request.method == "POST":
        print("in test")

        img = Image.open(request.files['file'])
        img = np.array(img)
        # plt.imshow(img)
        img = cv2.cvtColor(np.array(img), cv2.COLOR_BGR2RGB)
        # print(img)
        a = ranking_module(img)
        # print(a)
        data["filename_list"] = []
        count = 0
        for i in a:
            data["filename_list"].append(i)
            count+=1
            if count >= 10:
               break

        # data["filename_list"].append('topcit.png')
        # data["filename_list"].append('양동욱_21.jpg')

    return jsonify(data)

class Config:

	def __init__(self):

		# Print the process or not
		self.verbose = True

		# base network --> Todo! ResNet 으로 바꿔야함
		self.network = 'vgg'

		# Setting for data augmentation
		self.use_horizontal_flips = False
		self.use_vertical_flips = False
		self.rot_90 = False

		# Anchor box scales
    	# If im_size is smaller, anchor_box_scales should be scaled
	    # Original anchor_box_scales in the paper is [128, 256, 512]
		self.anchor_box_scales = [64, 128, 256]

		# Anchor box ratios
		self.anchor_box_ratios = [[1, 1], [1./math.sqrt(2), 2./math.sqrt(2)], [2./math.sqrt(2), 1./math.sqrt(2)]]

		# Size to resize the smallest side of the image
		# Original setting in paper is 600. Set to 300 in here to save training time
		self.im_size = 300

		# image channel-wise mean to subtract
		self.img_channel_mean = [103.939, 116.779, 123.68]
		self.img_scaling_factor = 1.0

		# number of ROIs at once --> Todo! If ResNet, RoI will be 3.
		self.num_rois = 4

		# stride at the RPN (this depends on the network configuration)
		self.rpn_stride = 16

		self.balanced_classes = False

		# scaling the stdev
		self.std_scaling = 4.0
		self.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]

		# overlaps for RPN --> Todo! If ResNet, IoU can be reduced. (what value? : 아마 0.5도 괜찮을듯 논문에서 본 적 있음)
		self.rpn_min_overlap = 0.3
		self.rpn_max_overlap = 0.7

		# overlaps for classifier ROIs
		self.classifier_min_overlap = 0.1
		self.classifier_max_overlap = 0.5

		# placeholder for the class mapping, automatically generated by the parser
		self.class_mapping = None

		self.model_path = None

"""# VGG Model"""

def get_img_output_length(width, height):
    def get_output_length(input_length):
        return input_length//16

    return get_output_length(width), get_output_length(height)

def nn_base(input_tensor=None, trainable=False):

    input_shape = (None, None, 3)

    if input_tensor is None:
        img_input = Input(shape=input_shape)
    else:
        if not K.is_keras_tensor(input_tensor):
            img_input = Input(tensor=input_tensor, shape=input_shape)
        else:
            img_input = input_tensor

    bn_axis = 3

    # Block 1
    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)
    x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)

    # Block 2
    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)
    x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)

    # Block 3
    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)
    x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)

    # Block 4
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)

    # Block 5
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)
    x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)
    # x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)

    return x

"""# Load Config Pickle"""

"""# Image Preprocessing"""

def format_img_size(img, C):
	""" formats the image size based on config """
	img_min_side = float(C.im_size)
	(height,width,_) = img.shape

	if width <= height:
		ratio = img_min_side/width
		new_height = int(ratio * height)
		new_width = int(img_min_side)
	else:
		ratio = img_min_side/height
		new_width = int(ratio * width)
		new_height = int(img_min_side)
	img = cv2.resize(img, (new_width, new_height), interpolation=cv2.INTER_CUBIC)
	return img, ratio

def format_img_channels(img, C):
	""" formats the image channels based on config """
	img = img[:, :, (2, 1, 0)]
	img = img.astype(np.float32)
	img[:, :, 0] -= C.img_channel_mean[0]
	img[:, :, 1] -= C.img_channel_mean[1]
	img[:, :, 2] -= C.img_channel_mean[2]
	img /= C.img_scaling_factor
	img = np.transpose(img, (2, 0, 1))
	img = np.expand_dims(img, axis=0)
	return img

def format_img(img, C):
	""" formats an image for model prediction based on config """
	img, ratio = format_img_size(img, C)
	img = format_img_channels(img, C)
	return img, ratio

#print(len(load_pickles.keys()))
#print(load_pickles.items())

"""# Demo
 - Tested ranking module by query image
"""

#print(load_pickles.keys())

"""# Version 2 : Input is pixel data & not exists in database"""

def ranking_module(query) :
    global count
    global C
    global model_backbone
    global load_pickles
    count += 1

    if count == 1:
        base_path = '/ENGN/'
        #test_path = 'drive/My Drive/capstone/retrieval_test/retrieval_annotations.txt'  # Test data (annotation file)
        test_base_path = '/ENGN/retrieval_test'  # Directory to save the test images
        config_output_filename = os.path.join(base_path, 'model_vgg_config.pickle')
        pickles = 'backbone_pickles'
        with open(config_output_filename, 'rb') as f_in:
            C = pickle.load(f_in)
        # turn off any data augmentation at test time
        C.use_horizontal_flips = False
        C.use_vertical_flips = False
        C.rot_90 = False
        """# Construct and Load the Model"""
        num_features = 512
        input_shape_img = (None, None, 3)
        img_input = Input(shape=input_shape_img)
        # define the base network *Todo : (VGG here, can be Resnet50 ...)*
        shared_layers = nn_base(img_input, trainable=True)
        # model backbone
        model_backbone = Model(img_input, shared_layers)
        model_backbone.load_weights('/ENGN/model_frcnn_vgg.hdf5', by_name=True)
        # model_backbone.load_weights(C.model_path, by_name=True)

        # model backbone
        '''
        Load Pickles
         - predict 512 values using backbone model
         - saved these values as pickle file
        '''
        load_pickles = cPickle.load(open(os.path.join(test_base_path, pickles), "rb", True))

    query_class=''
    X, ratio = format_img(query, C)
    X = np.transpose(X, (0, 2, 3, 1))
    # 변형부분
    result=model_backbone.predict(X)
    query_pooling = np.max(result[0], axis=(0, 1))
    result={}

    for filepath, value in load_pickles.items() :
        cosine=spatial.distance.cosine(query_pooling, value['avg_pooling'])
        result[filepath] = [cosine, value['id']]

    cls_count={'rose' : 0, 'tiger' : 0, 'dragon' : 0, 'geometric' : 0}
    temp_result=sorted(result.items(), key=lambda x: x[1][0])

    for i in temp_result[:20] :
        cls_count[load_pickles[i[0]]['cls']]+=1
    # max_key = max(cls_count.items(), key=lambda k: k[1])
    # print(cls_count)
    query_class=sorted(cls_count.items(), key=lambda x: x[1], reverse=True)
    # print(query_class)
    for filename, value in result.items() :
        alpha = 1 if cls_count[load_pickles[filename]['cls']]==0 else cls_count[load_pickles[filename]['cls']]
        result[filename][0] +=(20/alpha) * 0.08

        # if load_pickles[filename]['cls'] == query_class[0][0] :

        #     continue
        # elif load_pickles[filename]['cls'] == query_class[1][0] :
        #     result[filename][0] *= 2
        #     continue
        # else :
        #     result[filename][0] *= 2.5
        #     continue

    result=sorted(result.items(), key=lambda x: x[1])
    #print(query_class)
    # print(result)

    img=query
    # plt.figure(figsize=(24,20))
    # plt.imshow(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))
    # plt.subplot(653),plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)),plt.title('query image')
    # frame   = plt.gca()
    # frame.axes.get_xaxis().set_visible(False)
    # frame.axes.get_yaxis().set_visible(False)
    # k=0
    # for i in result :
    #     print(i[1][1])
    #     # print(i[1][0])
    # for i in result[:20] :

    #     img = cv2.imread(i[0])

    #     plt.subplot(6,5,k+6),plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)), plt.title("rank " + str(k+1) + " : " + str(i[1][0]))
    #     k+=1
    #     frame   = plt.gca()
    #     frame.axes.get_xaxis().set_visible(False)
    #     frame.axes.get_yaxis().set_visible(False)
    id_output=[x[1][1] for x in result]

    return id_output

count = 0
C = None
model_backbone = None
load_pickles = None

if __name__ == '__main__':
    app.run(host='0.0.0.0')


import pandas as pd

from sklearn.model_selection import train_test_split

from sklearn.preprocessing import MultiLabelBinarizer

def SplitData():
    columns = ["Post", "Seek", "medical_condition", "medical_test", "medication", "insurance",
           "diet", "exercise", "ask_for_advice", "other"]
    df = pd.read_csv("Data/dataset.csv", delimiter="^")

    df[["Seek","medical_condition", "medical_test", "medication", "insurance",
                "diet","exercise", "ask_for_advice", "other"]] = df[["Seek","medical_condition",
                                                                     "medical_test", "medication", "insurance",
                "diet","exercise", "ask_for_advice", "other"]].astype(float).fillna(0.0)
    y_seq2seq = []
    y_sequential = []
    for index, row in df.iterrows():
        temp1 =[]
        temp = []
        temp.append(0.0)
        for i in range(1, len(columns)):
            #if row[columns[i]] == 1:
            temp.append(row[i])
            if row[columns[i]] == 1:
                temp1.append(columns[i])
        temp.append(0.0)
        y_seq2seq.append(temp) #, [0.0]*(len(columns)-1) + [1.0]])
        y_sequential.append(temp1)

    #m = MultiLabelBinarizer()
    #Y = m.fit_transform(y)
    #pdb.set_trace()

    #Y = np.expand_dims(y,1)
    Y = y_seq2seq
    df["Frames_seq2seq"] = Y
    df["Frames_sequential"] = y_sequential
    train,test = train_test_split(df, test_size=0.15)
    train.to_pickle("Data/train.pkl")
    test.to_pickle("Data/test.pkl")

SplitData()


from flask import Flask, request, Response, render_template,jsonify, send_file
import random
from output import get_output
import requests
import os
from create import create_bq_results, create_menu_hours, create_store_status
import csv

app = Flask(__name__)

# Store the report generation status and data
report_data = {}

@app.route('/trigger_report')
def trigger_report():
    create_bq_results.create_table()
    create_menu_hours.create_table()
    create_store_status.create_table()
    report_id = generate_report_id()
    report_data[report_id] = {'status': 'Running'}
    report = get_output.get_output()
    report_data[report_id] = {'status': 'Complete', 'data': report}
    return jsonify({'report_id': report_id})

@app.route('/get_report', methods=['GET'])
def get_report():
    # Check if the report_id exists in the report_data dictionary
    json_data = request.json
    report_id = json_data.get('report_id')
    if report_id in report_data:
        report_status = report_data[report_id]['status']
        print(report_status)
        if report_status == 'Complete':
            current_dir = os.path.abspath(os.path.dirname(__file__))
            csv_location = os.path.join(current_dir, report_data[report_id]['data'])
            with open(csv_location, 'r') as file:
                csv_data = csv.DictReader(file)
                # Convert CSV data to a list of dictionaries
                data = [row for row in csv_data]
            # Set the appropriate MIME type for CSV
            mime_type = 'text/csv'

            # Return the CSV file as a response
            # return jsonify({'status': 'Complete', 'data':send_file(csv_location, mimetype=mime_type, as_attachment=True)})
            return jsonify({'status': 'Complete', 'data': data})
            # return send_file(report_data[report_id]['data'], as_attachment=True)
        else:
            return jsonify({'status': 'Running'})
    else:
        return jsonify({'status': 'Invalid report ID'})

@app.route('/')
def index():
    return render_template('index.html')

def generate_report_id():
    # Generate a random report ID
    return ''.join(random.choices('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', k=8))

if __name__ == '__main__':
    app.run(debug=True)


from smbus2 import SMBus
import Adafruit_MCP9808.MCP9808 as MCP9808
import time

bus = SMBus(1)

#Address of I2C devices
devices = []

#Sorted address of I2C devices
temps  = []
lights = []
others = []

#Sensors created using Adafruit library
sensorsT = []

#SRF08 REQUIRES 5V
def scan(force=False):
    devices = []
    for addr in range(0x03, 0x77 + 1):
        read = SMBus.read_byte, (addr,), {'force':force}
        write = SMBus.write_byte, (addr, 0), {'force':force}

        for func, args, kwargs in (read, write):
            try:
                with SMBus(1) as bus:
                    data = func(bus, *args, **kwargs)
                    devices.append(addr)
                    break
            except OSError as expt:
                if expt.errno == 16:
                    # just busy, maybe permanent by a kernel driver or just temporary by some user code
                    pass
    return devices

def taxonomy(devices):
    global lights,temps,others
    for d in devices:
        if d == 57:
            lights.append(d)
        elif d>=24 and d<=31:
            temps.append(d)
        else:
            others.append(d)

def readLight(address):
    bus.write_byte_data(address, 0x08, 0x03)
    time.sleep(0.05)
    val = bus.read_word_data(address, 0xAC)
    return val

#Main

devices = scan(force=True)
taxonomy(devices)

for d in temps:
    sensor = MCP9808.MCP9808(address=d, busnum=1)
    sensorsT.append({d, sensor})
    sensor.begin()

print("Lights :")
print(lights)
print("Temps :")
print(temps)

while True:
    #write(0x51)
    time.sleep(2)
    print("Temperatures : ")
    for ad, t in sensorsT:
        print( str(ad) + " : " + str(t.readTempC()) )

    print("Luminosities : ")
    for l in lights:
        print( str(l) + " : " + str(readLight(l)) )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 16 10:35:09 2022

@author: kamila
"""
import plotly.express as px
import plotly.graph_objects as go
import dash
from dash import dcc, html, no_update, ctx
from dash.dependencies import Input, Output, State, MATCH, ALL
import dash_bootstrap_components as dbc
import geopandas as gpd
import pandas as pd
import plotly.io as pio
import json
from datetime import datetime
import numpy as np
import base64
import io
import ast
import sys

pd.set_option('display.max_columns', None)
pd.set_option('display.width', 1300)

# plot postcodes
pio.renderers.default='browser'

# import df
pc_sales_df = gpd.read_file('pc_sales_df.geojson')

# join ship from locations
delivery_loc = pd.read_csv('pc_shipfrom_loc.csv')
delivery_loc['Row Labels'] = delivery_loc['Row Labels'].astype('str')
delivery_loc['Row Labels'] = delivery_loc['Row Labels'].apply(lambda x: '0' + x if x.startswith('8') else x)
delivery_loc.replace(0, False, inplace = True)
delivery_loc.replace(1, True, inplace = True)

pc_sales_df = pc_sales_df.merge(delivery_loc, left_on = 'poa_code21', right_on = 'Row Labels', how = 'left')

# ready geometries

pc_sales_df['geometry'] = pc_sales_df.to_crs(pc_sales_df.estimate_utm_crs()).simplify(500).to_crs(pc_sales_df.crs) # simplify boundaries to 500m
pc_sales_df = pc_sales_df.to_crs( epsg = 4326) # change to lat/long
pc_sales_df.set_index('poa_code21', inplace = True)

df = pc_sales_df
df['terr_colour'] = np.NaN

# store data WA bunnings
store_data = pd.read_csv(r'20221121_SF_latest_add.csv')
store_data['total_tw_purchases'] = store_data['total_tw_purchases'].astype(float)

# Define colors for each group
color_map = {
    'BUNNINGS': 'blue',
    'INDEPENDENTS': 'red'
}

# Iterate over the 'group' column and assign colors accordingly
colors = [color_map[group] for group in store_data['group']]

store_markers = go.Scattermapbox(
    lat=store_data['store_lat'],
    lon=store_data['store_long'],
    mode='markers',
    marker=dict(
        size=store_data['total_tw_purchases'],
        sizemode='diameter',
        sizeref=10000,
        color=colors,
        colorscale='Viridis',  # Add a colorscale to enable color differentiation
        opacity=0.7
    ),
    text=store_data['cust name'],
    name='All'
)

# store_data_indy = pd.read_csv(r'C:\Users\KAmbrozewicz\OneDrive - UMCOS NEWCOMBE PTY LIMITED\GIS\20221121_SF_latest_add.csv')

# store_heatmap = go.Figure(
#     data=[
#         go.Densitymapbox(
#             lat=store_data_indy['lat'],
#             lon=store_data_indy['long'],
#             radius=60 ,
#             name = 'Heatmap',
#             colorscale='Viridis',  # Specify a discrete color scale
#             zmin=1,  # Set the minimum value for color scaling
#             zmax=5  # Set the maximum value for color scaling
#         )
#     ]
# )

# filter by state to speed up plotting

state_coords = {'NT' : [-19.491411, 132.550964],
                'NSW' : [-33.872762037132375, 147.22963557432993],
                'VIC' : [-37.020100, 144.964600],
                'QLD' : [-20.917574, 142.702789],
                'SA' : [-30.000233, 136.209152],
                'WA' : [-25.953512, 117.857048],
                'TAS' : [-41.640079, 146.315918]
                }

# initialise variables
df_state = df[df.codestte == 'WA']

fig = px.choropleth_mapbox(df_state,
                           geojson = df_state.geometry,
                           locations = df_state.index,
                           opacity = 0.2,
                           center = {'lat' : -25.953512, 'lon' : 117.857048},
                           zoom = 4.5,
                           height = 1800,
                           width = 1600,
                           mapbox_style="carto-positron",
                           # uirevision = 'Retain user zoom preferences'
)

fig.update_layout(
    legend=dict(
        yanchor="top",
        y=0.9,
        xanchor="left",
        x=0.01,
        font=dict(
            size=12
        )
    )
)
fig.add_trace(store_markers)

# app
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])

server = app.server

app.layout = html.Div(
    [
     dbc.Row(
        #  dbc.Col(
        #      [
        #          html.Div(
        #              [
        #                  html.Pre('Select a State', style = {'paddingRight' : '20px', 'paddingTop' : '10px', 'paddingBottom' : '0px'}),

        #                  dcc.Dropdown(
        #                              id = 'select_state',
        #                               options = [{'label' : i, 'value' : i} for i in pc_sales_df.codestte.unique()],
        #                               value = 'NSW',
        #                               multi = True,
        #                               style = {'width' : 400}

        #                 )
        #             ], style = {'paddingLeft' : '70px', 'paddingTop' : '10px', 'display' : 'flex'}
        #         )
        #     ]
        # )
    ),

     dbc.Row(
         [
             dbc.Col(
                 [
                    dcc.Graph(id = 'graph', figure = fig)
                 ], width = 9

            ),

             dbc.Col(
                 [
                     dbc.Row(
                         [
                             html.Div(
                                 [
                                     html.Br(),

                                        html.Div(
                                                html.Pre('Select a State',
                                                        style = {
                                                                  'font-family' : 'arial',
                                                                  'font-size' : '17px',
                                                                  'paddingLeft' : '10px',
                                                                  'paddingTop' : '10px',
                                                                  'paddingBottom' : '0px'
                                                         }
                                                ), style = {'width' : '160px', 'paddingRight' : '10px'}
                                        ),

                                     dcc.Dropdown(
                                                 id = 'select_state',
                                                  options = [{'label' : i, 'value' : i} for i in pc_sales_df.codestte.unique()],
                                                  value = 'WA',
                                                  multi = True,
                                                  style = {'width' : 310}

                                    )
                                ], style = {'paddingTop' : '50px', 'display' : 'flex'}
                            ),
                         ]
                    ),
                     dbc.Row(
                         [
                             html.Div(
                                 [

                                     html.Button('Add new legend entry',
                                                 id = 'btn_add_legend_entry',
                                                 style = {
                                                         'width' : 150,
                                                         'height' : 75,
                                                         'backgroundColor' : 'rgb(177,221,140)',
                                                         'marginRight' : '10px',
                                                         'borderWidth' : '1px',
                                                         'borderStyle' : 'solid',
                                                         'borderRadius' : '3px',
                                                         'borderColor' : 'rgb(174,174,174)',
                                                }
                                    ),

                                     html.Button('Download map and legend',
                                                 id = 'btn_download_map',
                                                 style = {
                                                         'width' : 150,
                                                         'height' : 75,
                                                         'margin-right' : '10px',
                                                         'borderWidth' : '1px',
                                                         'borderStyle' : 'solid',
                                                         'borderRadius' : '3px',
                                                         'borderColor' : 'rgb(174,174,174)',
                                                         'background-color' : 'rgb(242,242,242)'
                                                }
                                    ),

                                     html.Div(
                                             dcc.Upload('Click to import data',
                                                        id = 'import_data',
                                                        style = {
                                                                'textAlign' : 'center',
                                                                'margin' : '10px',
                                                                'cursor' : 'pointer'
                                                        }

                                            ), style = {'width' : 150,
                                                        'height' : 75,
                                                        'borderWidth' : '1px',
                                                        'borderStyle' : 'solid',
                                                        'borderRadius' : '3px',
                                                        'borderColor' : 'rgb(174,174,174)',
                                                        'background-color' : 'rgb(242,242,242)'
                                                }
                                    ),

                                 ], style = {'display' : 'flex', 'paddingBottom' : '10px'}
                            ),
                        ]
                    ),

                    html.Div(id = 'test_area'),

                    dcc.Store(id = 'legend_triggered', storage_type = 'memory'), # which legend was last triggered

                    dcc.Store(id = 'map_legend_data', data = {}, storage_type = 'memory'), # map data stored for download

                    dcc.Store(id = 'map_upload', storage_type = 'memory'),

                    dcc.Store(id = 'nn_click_count', data = 0, storage_type = 'memory'),

                    dcc.Store(id = 'legend_data', storage_type = 'memory'),

                    dcc.Download(id = 'download_map_data'),

                    #html.Button('test button', id = 'test_button', style = {'width' : 200, 'height' : 75}),

                    html.Div(id = 'legend_container', children = []),

                    html.Div(id = 'test_import_area'),

                    html.Div(id = 'legend_label_test_area', children = [])

                 ]
            )
        ]
    )
    ]
)
# access legend labels, colours
@app.callback(
    Output('legend_data', 'data'),
    #Output('legend_label_test_area', 'children'),
    Input({'type' : 'colour_label', 'index' : ALL},'value'),
    State({'type' : 'colour_label', 'index' : ALL}, 'value'),
    State({'type' : 'colour_input', 'index' : ALL}, 'value'),
    prevent_initial_call = True
    )
def get_legend_label(n_clicks, legend_labels, legend_colour):
    legend_dict = dict(zip(legend_colour, legend_labels))
    return legend_dict #, html.Pre('get_legend_label function output: ' + str(legend_dict))

# import data
@app.callback(
    Output('map_upload', 'data'),
    #Output('test_import_area', 'children'), # populate div while debugging
    Input('import_data', 'contents'),
    prevent_initial_call = True
)
def parse_contents(contents):
    content_type, content_string = contents.split(',')
    decoded_content = base64.b64decode(content_string)
    decoded_str = decoded_content.decode('utf-8')
    return decoded_str #, html.Pre('output from parse_contents function: ' + decoded_str[:50])

# download data
@app.callback(
    Output('download_map_data', 'data'),
    # Output('legend_label_test_area', 'children'),
    Input('btn_download_map', 'n_clicks'),
    State('map_legend_data', 'data'),
    State('legend_data', 'data'),
    prevent_initial_call = True
)

def download_data(n_clicks, data, legend_data): # this version is outputting map and legend data as list of dictionaries
    if n_clicks is not None:
        download_legend_data = json.dumps([legend_data, data])
        download_data = json.dumps(data)
        content_dict = {'content' : download_legend_data}
    return dict(content_dict, filename = 'mapdata.text') #, html.Pre(str(download_legend_data))

# add new legend entry
@app.callback(
    Output('legend_container', 'children'),
    # Output('btn_add_legend_entry', 'n_clicks'),
    Output('nn_click_count', 'data'),
    Input('btn_add_legend_entry', 'n_clicks'),
    Input('map_upload', 'data'),
    State('legend_container', 'children'),
    State('nn_click_count', 'data'),
    prevent_initial_call = True
)
def add_legend(n_clicks, map_upload, container, nn_clicks):
    if map_upload is None:
        if n_clicks is None:
            return container, nn_clicks
        new_legend = html.Div(
            id={'type': 'legend_div', 'index': nn_clicks},
            children=[
                dbc.Input(
                    id={'type': 'colour_input', 'index': nn_clicks},
                    type='color',
                    style={'width': 75, 'height': 50}
                ),
                dbc.Input(
                    id={'type': 'colour_label', 'index': nn_clicks},
                    type='text',
                    placeholder='Enter a sales territory name',
                    style={'width': 395, 'height': 50}
                )
            ],
            style={'display': 'flex'}
        )
        nn_clicks += 1
        container.append(new_legend)

        return container, nn_clicks

    else:
        json_list = json.loads(map_upload)
        pc_colour_d = json_list[0]
        container.clear()  # Clear existing legend entries
        for col, label in pc_colour_d.items():
            new_legend = html.Div(
                id={'type': 'legend_div', 'index': nn_clicks},
                children=[
                    dbc.Input(
                        id={'type': 'colour_input', 'index': nn_clicks},
                        value=str(col),
                        type='color',
                        style={'width': 75, 'height': 50}
                    ),
                    dbc.Input(
                        id={'type': 'colour_label', 'index': nn_clicks},
                        type='text',
                        value=label,
                        style={'width': 395, 'height': 50}
                    )
                ],
                style={'display': 'flex'}
            )
            container.append(new_legend)
            nn_clicks += 1
        return container, nn_clicks

# colour selected listener
@app.callback(
    Output('legend_triggered', 'data'),
    # Output('test_area', 'children'),
    Input({'type' : 'legend_div', 'index' : ALL}, 'n_clicks'),
    prevent_initial_call = True
    )
def colour_listener(colourLastSelected):
    return ctx.triggered_id.index #, html.Pre(ctx.triggered_id.index)

# update map
@app.callback(
    Output('graph', 'figure'),
    Output('graph', 'clickData'),
    Output('map_legend_data', 'data'),
    #Output('test_area', 'children'),
    Output('map_upload', 'clear_data'),
    Input('map_legend_data', 'data'),
    Input('graph', 'clickData'),
    Input('select_state', 'value'),
    State({'type' : 'colour_input', 'index' : ALL}, 'value'),
    State('legend_triggered', 'data'),
    Input('map_upload', 'data'),
    prevent_initial_call = True
    )
def update_map(postcode_colour_d, selectPC, selectedState, selectedColor, legendTriggered, mapUpload):
    clear_map_data = False

    if mapUpload is not None:
        json_str_list = json.loads(mapUpload)
        json_str = json_str_list[1]
        postcode_colour_d.update(json_str)
        clear_map_data = True

    if selectPC is not None:
        postcode = selectPC['points'][0]['location']

        if postcode in postcode_colour_d:
            postcode_colour_d.pop(postcode)
        else:
            if len(selectedColor) == 0 or None in selectedColor:
                global fig
                return fig, None, postcode_colour_d, clear_map_data
            else:

                triggeredColor = selectedColor[legendTriggered]
                postcode_colour_d[postcode] = triggeredColor

    postcode_list = list(postcode_colour_d.keys())

    if type(selectedState) == str:
        df_state = df[df.codestte == selectedState]

    else:
        df_state = df[df.codestte.isin(selectedState)]

    fig = px.choropleth_mapbox(df_state,
                                geojson = df_state.geometry,
                                locations = df_state.index,
                                opacity = 0.2,
                                #center = {'lat' : state_coords[selectedState][0], 'lon' : state_coords[selectedState][1]},
                                center = {'lat' : -25.953512, 'lon' : 117.857048},
                                zoom = 4.5,
                                height = 1800,
                                width = 1600,
                                mapbox_style="carto-positron"
    )

    selected_df = df_state.loc[postcode_list, :]
    selected_df.reset_index(inplace = True)
    selected_df['terr_colour'] = selected_df['poa_code21'].apply(lambda x: postcode_colour_d.get(x))
    selected_df.set_index('poa_code21', inplace = True)

    for colour in selected_df.terr_colour.unique():
        dff = selected_df[selected_df.terr_colour == colour]
        fig.add_trace(
            px.choropleth_mapbox(dff,
                                  geojson = dff.geometry,
                                  locations = dff.index,
                                  color = dff.terr_colour,
                                  color_discrete_map=({colour : colour}),
                                  opacity = 0.7
                                  ).data[0]
        )

    fig.update_layout(uirevision = 'Retain user zoom preferences')
    fig.add_trace(store_markers)

    #debug_output = 'index triggered: ' + str(legendTriggered) + ' color selected: ' +str(selectedColor) +  'postcode list: ' + str(postcode_list) + ' ' + str(postcode_colour_d)

    return fig, None, postcode_colour_d, clear_map_data

if __name__ == "__main__": app.run_server(debug=False, host='127.0.0.1', port=8050, dev_tools_hot_reload = True)


import math
import os
import random
import re

import bpy
from mathutils import Color, Euler, Vector
import numpy as np
from math import radians,tan,cos,sin

class Blender_helper:

    def __init__(self) -> None:
        pass

    def clear_scene(self):
        # Clear existing objects
        # Delete all objects
        bpy.ops.object.select_all(action='DESELECT')
        bpy.ops.object.select_by_type(type='MESH')
        bpy.ops.object.delete()

        # Delete all materials
        for material in bpy.data.materials:
            bpy.data.materials.remove(material)

        # Delete all cameras
        for camera in bpy.data.cameras:
            bpy.data.cameras.remove(camera)

        # Delete all lights
        for light in bpy.data.lights:
            bpy.data.lights.remove(light)

        # Delete all curves (bezier circles, etc.)
        for curve in bpy.data.curves:
            bpy.data.curves.remove(curve)

        # Delete all images
        for image in bpy.data.images:
            bpy.data.images.remove(image)

        # Delete all textures
        for texture in bpy.data.textures:
            bpy.data.textures.remove(texture)

        # Delete all nodes in node trees
        for material in bpy.data.materials:
            if material.use_nodes:
                material.node_tree.nodes.clear()

        # Delete all node groups
        for node_group in bpy.data.node_groups:
            bpy.data.node_groups.remove(node_group)

        return None

    def add_basic_scene(self):
        """
        This function adds  basic setup like, camera, light and floor for the objects

        use the functions below
        """

        # add background plane
        background_plane = self.add_background_plane(
            plane_size=5, name='Background_plane')

        # add light source and set it to desired place
        light_source = self.add_light_source(
            type='sun', obj_name='Sun', shadow=True)

        # add camera and set it to desired place
        camera = self.add_camera(camera_name='Camera')

        # add curves for following the path
        camera_track = self.add_beizer_curve(
            name='camera_track', location=(0, 0, 2.5), scale=(0, 0, 0))
        light_track = self.add_beizer_curve(
            name='light_track', location=(0, 0, 3), scale=(0, 0, 0))

        # Set the follow path constraints
        camera.constraints.new(type='FOLLOW_PATH')
        camera.constraints['Follow Path'].target = camera_track

        light_source.constraints.new(type='FOLLOW_PATH')
        light_source.constraints['Follow Path'].target = light_track

        return background_plane, light_source, camera, camera_track, light_track

    def add_regression_scene(self):
        # add background plane
        background_plane = self.add_background_plane(
            plane_size=5, name='Background_plane')

        # add light source and set it to desired place
        light_source = self.add_light_source(
            type='sun', obj_name='Sun', shadow=True)

        # add camera and set it to desired place
        camera = self.add_camera(camera_name='Camera')

        # add curves for following the path
        light_track = self.add_beizer_curve(
            name='light_track', location=(0, 0, 3), scale=(0, 0, 0))
        light_source.constraints.new(type='FOLLOW_PATH')
        light_source.constraints['Follow Path'].target = light_track

        camera_track = self.add_nurbs_path(name='camera_track')
        camera.constraints.new(type='FOLLOW_PATH')
        camera.constraints['Follow Path'].target = camera_track

        return background_plane, light_source, camera, camera_track, light_track

    def add_nurbs_path(self, name):
        bpy.ops.curve.primitive_nurbs_path_add(enter_editmode=False,align='WORLD',location=(0, 0, 1),rotation=(0,1.5707,0),scale=(1, 1, 1))

        bpy.data.objects['NurbsPath'].name = str(name)
        return bpy.data.objects[str(name)]

    def add_beizer_curve(self, name, location, scale):
        bpy.ops.curve.primitive_bezier_circle_add(enter_editmode=False,
                                                  align='WORLD',
                                                  location=location,
                                                  scale=scale
                                                  )
        bpy.data.objects['BezierCircle'].name = str(name)
        return bpy.data.objects[str(name)]

    def set_random_curve_height(curve_object_name, min_value, max_value):
        """ This function sets the height of the path curve for randomness/variation
        """
        z_value = round(random.uniform(min_value, max_value), 2)
        bpy.data.objects[str(curve_object_name)].location[2] = z_value

    def track_object(self, tracking_object_name, object_to_track):
        """ Function to make camera track the objects
        """
        tracking_object = bpy.data.objects[str(tracking_object_name)]
        if not str('Track To') in tracking_object.constraints.keys():
            tracking_object.constraints.new(type="TRACK_TO")
            tracking_object.constraints['Track To'].target = object_to_track
        else:
            tracking_object.constraints['Track To'].target = object_to_track

    def add_background_plane(self, plane_size, name):
        """
        This function adds the background plane to the scene along with a default material.
        """
        bpy.ops.mesh.primitive_plane_add(
            size=plane_size, enter_editmode=False, align='WORLD', location=(0, 0, 0))
        bpy.data.objects['Plane'].name = name
        return bpy.data.objects[str(name)]

    def add_camera(self, camera_name: str):
        """
        Adds a camera to the origin of the scene

        Keyword arguments:
        camera_name -- Name for the camera: str
        """
        bpy.ops.object.camera_add(
            location=(0.0, 0.0, 0.0), rotation=(0.0, 0.0, 0.0))
        bpy.context.object.data.lens = 50
        bpy.context.object.data.name = str(camera_name)
        bpy.context.scene.camera = bpy.context.object
        return bpy.data.objects[str(camera_name)]

    def add_light_source(self, type: str, obj_name: str, shadow: bool):
        """
        Adds a light source with respective type to the origin of the scene.

        Keyword arguments:
        type -- The type of light source, default SUN, other: POINT,AREA,SPOT
        obj_name -- Name for the object: str
        shadow -- Enable or Disable shadows: bool

        returns: Light object
        """
        bpy.ops.object.light_add(
            type=str(type).upper(), radius=1, location=(0, 0, 0))
        bpy.context.object.data.name = str(obj_name)
        bpy.context.object.data.energy = 5
        bpy.context.object.data.use_shadow = shadow
        bpy.context.object.data.use_contact_shadow = shadow
        return bpy.data.objects[str(obj_name)]

    def set_camera(self, obj_name: str, position: tuple, rotation: tuple):
        """
        This function sets the camera to the desired location and orientation.
        """
        bpy.data.cameras[str(obj_name)].location = position
        bpy.data.cameras[str(obj_name)].Rotation = rotation

    def set_light_source(self, obj_name: str, position: tuple, rotation: tuple):
        """
        Sets the light source to the desired loacation,orientation and energy values.

        Keyword arguments:
        obj_name -- Name of the light object: str
        position -- Position values of the light object: (float,float,float)
        rotation -- Rotation values of the light object: (float,float,float)
        """
        bpy.data.lights[str(obj_name)].location = position
        bpy.data.lights[str(obj_name)].Rotation = rotation

    def get_object_names(self, background_plane_name, camera_track, light_track):
        """
        returns list of object names present in the scene after removing unnecessary things.

        Keyword arguments:
        background_plane_name--name of the background plane object: str
        """

        obj_names = bpy.context.scene.objects.keys()
        for name in bpy.data.cameras.keys():
            obj_names.remove(name)
        for name in bpy.data.lights.keys():
            obj_names.remove(name)

        obj_names.remove(str(background_plane_name))
        obj_names.remove(str(camera_track))
        obj_names.remove(str(light_track))
        return sorted(obj_names)

    def hide_objects(self, obj_names, hide: bool):
        """
        hides the objects present in the scene during rendering.
        """
        for name in obj_names:
            bpy.context.scene.objects[name].hide_render = hide

    def random_placement(self,obj, camera, min_distance_factor):
        scene = bpy.context.scene
        # Get the current camera field of view in degrees if not provided

        fov_radians = camera.data.angle
        print(f"\nFov radians is : {fov_radians} ")
        # Calculate camera height and width for the current FOV
        aspect_ratio = camera.data.sensor_width / camera.data.sensor_height
        aspect_ratio = scene.render.resolution_x/scene.render.resolution_y
        camera_z_loc = camera.matrix_world.decompose()[0][2]
        print(f"Camera Z location is : {camera_z_loc}")
        camera_height = 2 * camera_z_loc * tan(fov_radians / 2)
        camera_width = camera_height * aspect_ratio

        # Calculate the size of the object within the camera frame (e.g., 20% of camera width)
        object_size_percentage = 0.1
        object_width = object_size_percentage * camera_width

        print(f"Object width is {object_width}")
        # Calculate the maximum distance for the object based on its size and desired size within the frame
        max_distance = object_width / (2 * tan(fov_radians / 2))

        # Calculate the minimum distance based on a factor of the maximum distance
        min_distance = min_distance_factor * max_distance

        # Calculate random distance within the range
        print(f'Maximum distance is {max_distance}, Minimum distance is : {min_distance}')
        random_distance = random.uniform(min_distance, max_distance)

        # Calculate random angle (azimuth) around the camera
        random_angle = random.uniform(0, 2 * np.pi)

        # Calculate the object position relative to the camera
        random_x = random_distance * cos(random_angle)
        random_y = random_distance * sin(random_angle)
        random_z = obj.dimensions.z / 2

        print(f'Original x is {obj.location.x} : Random x is {random_x} Original y is : {obj.location.y} Random y is {random_y}')
        # Set the object location
        obj.location.x = random_x
        obj.location.y = random_y

    def import_objects_obj_format(self, models_dir, target_object_size, dataset_name):
        """
        adds objects from cad format(.obj format) along with their materials.

        models_dir: path for the models
        target_size: controls the size of the objects
        dataset_name: Name of the dataset, Robocup or YCB
        """

        object_names = []
        obj_files = []
        # Iterate through all subdirectories in the base path
        for root, dirs, files in os.walk(models_dir):
            # Iterate through all files in the current directory
            for file in files:
                # Check if the file is an OBJ file
                if file.endswith(".obj"):
                    # Construct the full path to the OBJ file
                    object_file = os.path.join(root, file)
                    obj_files.append(object_file)

        # Clear the scene
        bpy.ops.wm.read_factory_settings()
        self.clear_scene()

        for idx, object_file in enumerate(obj_files):

            if dataset_name == 'robocup':
                obj_name = object_file.split('/')[-1].split('.')[0]
            elif dataset_name == 'ycb':
                obj_name = "".join(object_file.split('/')[-3].split('_')[1::])

            # Import the OBJ file
            bpy.ops.import_scene.obj(filepath=object_file)

            # Set the active object to the imported object
            imported_object = bpy.context.selected_objects[-1]
            # Rename the new object
            imported_object.name = str(obj_name)
            object_names.append(imported_object.name)

            if imported_object is not None:
                # Get the maximum dimension of the object
                max_dimension = max(imported_object.dimensions)

                # Calculate the scale factor to achieve the target size
                scale_factor = target_object_size / max_dimension

                # Apply scale transformation to the object
                imported_object.scale = (
                    scale_factor, scale_factor, scale_factor)

                # Set the origin of the object to its center
                imported_object.select_set(True)
                bpy.context.view_layer.objects.active = imported_object
                bpy.ops.object.origin_set(
                    type='ORIGIN_CENTER_OF_VOLUME', center='BOUNDS')
                imported_object.location = (0.0, 0.0, 0.0)
                imported_object.rotation_euler = (0.0, 0.0, 0.0)
            else:
                print("Error: Object is None")

        return None

    def get_object(self, obj_name):
        """
        Returns the object present in the scene.
        """
        return bpy.context.scene.objects[obj_name]

    def set_random_rotation(self, obj_to_change):
        """
        Applies a random rotation to the given object.

        Keyword arguments:
        obj_to_change: blender object
        """

        random_rotat_values = [
            random.random()*2*math.pi, random.random()*2*math.pi, random.random()*2*math.pi]
        obj_to_change.rotation_euler = Euler(random_rotat_values, 'XYZ')

    def set_random_lighting(self, light_source_name, min_value, max_value):
        """
        Applies random light intensities to the scene.

        Keyword arguments:
        light_source_name:str
        min_value: float
        max_value: float
        """
        # bpy.data.lights[str(light_source_name)].energy = round(random.uniform(min_value,max_value),2)
        bpy.data.lights[str(light_source_name)].energy = random.uniform(
            min_value, max_value)

    def set_random_focal_length(self, camera_name, min_value, max_value):
        value = random.randint(min_value, max_value)
        bpy.data.cameras[str(camera_name)].lens = float(value)

    def set_random_background_color(self, obj_name):
        """Applies Materials randomly to the object, Changes specially the Principled BSDf Base color values for the given object's material.

        Keyword arguments:
        obj_name: str
        """
        material_to_change = bpy.data.objects[str(obj_name)].active_material
        # bpy.data.materials[material_name]
        color = Color()
        hue = random.random()  # Random hue between 0 and 1
        color.hsv = (hue, 0.85, 0.85)
        rgba = [color.r, color.g, color.b, 1]
        material_to_change.node_tree.nodes['Principled BSDF'].inputs[0].default_value = rgba

    def set_render_parameters(self, device, render_engine, res_x, res_y, num_samples):
        """Sets the active scene with the specified render parameters.

        Keyword arguments:
        device-- Type of the device CPU or GPU : str
        res_x-- resolution width of the image to render: int
        res_y-- resolution height of the image to render: int
        num_samples-- number of render samples
        """
        scene = bpy.context.scene
        scene.render.engine = str(render_engine)
        scene.cycles.device = str(device).upper()
        scene.render.resolution_x = int(res_x)
        scene.render.resolution_y = int(res_y)
        scene.render.resolution_percentage = 100
        scene.cycles.samples = num_samples
        scene.render.image_settings.file_format = "PNG"

    def get_texture_map_paths(self, texture_folder):
        """Returns paths for the images which can be used for image textures.

        Keyword arguments:
        texture_folder-- Path for the folder containing the images.
        """
        texture_dict = {
            "normal_map": None,
            "base_color": None,
            "disp_map": None,
            "metal_map": None,
            "roughness_map": None
        }

        files = os.listdir(path=texture_folder)
        for file in files:
            # Match the files and seperate
            nrm_match = re.search(r'\wnormal', file) or re.search(
                r'\wNRM', file) or re.search(r'\wnor', file)
            base_match = re.search(r'\wbasecolor', file) or re.search(
                r'\wCOL_VAR1', file) or re.search(r'\wdiff', file) or re.search(r'\wcol', file)
            disp_match = re.search(r'\wDISP_4K', file) or re.search(
                r'\wheight', file) or re.search(r'\wdisplacement', file) or re.search(r'\wdisp', file)
            metal_match = re.search(r'\wmetallic', file) or re.search(
                r'\wREFL', file) or re.search(r'\wmetal', file)
            rough_match = re.search(r'\wroughness', file) or re.search(
                r'\wGLOSS', file) or re.search(r'\wrough', file)

            if nrm_match:
                normal_map = os.path.join(texture_folder, file)
                texture_dict['normal_map'] = normal_map

            if base_match:
                base_color = os.path.join(texture_folder, file)
                texture_dict['base_color'] = base_color
            if disp_match:
                disp_map = os.path.join(texture_folder, file)
                texture_dict['disp_map'] = disp_map
            if metal_match:
                metal_map = os.path.join(texture_folder, file)
                texture_dict['metal_map'] = metal_map
            if rough_match:
                roughness_map = os.path.join(texture_folder, file)
                texture_dict['roughness_map'] = roughness_map
        return texture_dict

    def get_texture_paths(self, texture_dir):
        """Gets the paths for all the texture_folders from the main textures folder and returns them as a list

        Keyword arguments:
        texture_dir-- path for the directory containig the textures.
        """
        bnames = os.listdir(texture_dir)
        for i, cs in enumerate(zip(*bnames)):
            if len(set(cs)) != 1:
                break
        for _i, cs in enumerate(zip(*[b[::-1] for b in bnames])):
            if len(set(cs)) != 1:
                break
        texture_paths = [os.path.join(texture_dir, bname+'/')
                         for bname in bnames if not bname.endswith('.md')]
        return texture_paths

    def set_random_pbr_img_textures(self, textures_path, obj_name, scale):
        """Applies image textures randomly from the available images to the specified object.

        Keyword arguments:
        texture_paths-- list of paths for the texture folders.
        obj_name-- name of the object to change the materail/texture
        """

        texture_paths = self.get_texture_paths(textures_path)
        # print("Texture paths : ",texture_paths)
        texture_path = random.choice(texture_paths)

        material_name = str(texture_path.split('/')[-2])
        texture_dict = self.get_texture_map_paths(texture_folder=texture_path)

        # create a new material with the name.
        material = bpy.data.materials.new(
            name=material_name)  # Change name everytime
        material.use_nodes = True

        # Create the nodes for the material

        # Nodes for controlling the texture
        texture_coordinate = material.node_tree.nodes.new(
            type="ShaderNodeTexCoord")
        mapping_node = material.node_tree.nodes.new(type="ShaderNodeMapping")
        mapping_node.inputs['Scale'].default_value = (
            scale, scale, scale)  # Control the scale of the texture

        # Create vector nodes
        normal_map = material.node_tree.nodes.new(type="ShaderNodeNormalMap")
        displacement_map = material.node_tree.nodes.new(
            type="ShaderNodeDisplacement")
        bump_map = material.node_tree.nodes.new(type="ShaderNodeBump")
        invert_node = material.node_tree.nodes.new(type="ShaderNodeInvert")

        # Nodes for principled bsdf and output
        principled_bsdf = material.node_tree.nodes['Principled BSDF']
        material_output = material.node_tree.nodes['Material Output']

        # Connect the nodes
        material.node_tree.links.new(
            texture_coordinate.outputs['UV'], mapping_node.inputs['Vector'])
        # Nodes for image textures
        # Base color
        if texture_dict['base_color'] != None:
            base_color_img = material.node_tree.nodes.new(
                type="ShaderNodeTexImage")
            base_color_img.image = bpy.data.images.load(
                texture_dict['base_color'])
            material.node_tree.links.new(
                mapping_node.outputs['Vector'], base_color_img.inputs['Vector'])
            material.node_tree.links.new(
                base_color_img.outputs['Color'], principled_bsdf.inputs['Base Color'])

        # Normal map
        if texture_dict['normal_map'] != None:
            normal_img = material.node_tree.nodes.new(
                type="ShaderNodeTexImage")
            normal_img.image = bpy.data.images.load(texture_dict['normal_map'])
            material.node_tree.links.new(
                mapping_node.outputs['Vector'], normal_img.inputs['Vector'])
            material.node_tree.links.new(
                normal_img.outputs['Color'], normal_map.inputs['Color'])
            material.node_tree.links.new(
                normal_map.outputs['Normal'], principled_bsdf.inputs['Normal'])

        # Displacement map
        if texture_dict['disp_map'] != None:
            displacement_img = material.node_tree.nodes.new(
                type="ShaderNodeTexImage")
            displacement_img.image = bpy.data.images.load(
                texture_dict['disp_map'])
            material.node_tree.links.new(
                mapping_node.outputs['Vector'], displacement_img.inputs['Vector'])
            material.node_tree.links.new(
                displacement_img.outputs['Color'], displacement_map.inputs['Height'])
            material.node_tree.links.new(
                displacement_map.outputs['Displacement'], material_output.inputs['Displacement'])

        # Roughness map
        if texture_dict['roughness_map'] != None:
            roughness_img = material.node_tree.nodes.new(
                type="ShaderNodeTexImage")
            roughness_img.image = bpy.data.images.load(
                texture_dict['roughness_map'])
            material.node_tree.links.new(
                mapping_node.outputs['Vector'], roughness_img.inputs['Vector'])
            material.node_tree.links.new(
                roughness_img.outputs['Color'], invert_node.inputs['Color'])
            material.node_tree.links.new(
                invert_node.outputs['Color'], principled_bsdf.inputs['Roughness'])

        # Metal map
        if texture_dict['metal_map'] != None:
            metallic_img = material.node_tree.nodes.new(
                type="ShaderNodeTexImage")
            metallic_img.image = bpy.data.images.load(
                texture_dict['metal_map'])
            material.node_tree.links.new(
                mapping_node.outputs['Vector'], metallic_img.inputs['Vector'])
            material.node_tree.links.new(
                metallic_img.outputs['Color'], principled_bsdf.inputs['Metallic'])

        # Set material final output
        material.node_tree.links.new(
            principled_bsdf.outputs['BSDF'], material_output.inputs['Surface'])

        # Set the material to the object
        obj = bpy.data.objects[obj_name]

        if obj.data.materials:
            obj.data.materials.append(material)
            obj.active_material = bpy.data.materials[material_name]
        else:
            obj.data.materials.append(material)
            obj.active_material = material

        # print("List of materials : ", bpy.data.materials.keys())

        return material

    def add_blur_dof(self, blur_value,focus_background_name):
        """Adds blur effect to the images using depth of field parameter of the camera.

        Keyword arguments:
        focus_background_name: name of the backgrond plane
        """

        camera = bpy.data.objects['Camera']
        camera.data.dof.use_dof = True
        # camera.data.dof.focus_object = bpy.data.objects[str(focus_background_name)]
        camera.data.dof.focus_distance = blur_value

    def deform_objects(self, obj_to_deform):
        """ Deforms the object using simple deform modifer properties

        Keyword arguments:
            obj_to_deform: blender object to deform
        """

        if not 'simple_deform' in obj_to_deform.modifiers.keys():
            obj_to_deform.modifiers.new(
                name="simple_deform", type='SIMPLE_DEFORM')
            deform_modifier = obj_to_deform.modifiers['simple_deform']
        else:
            deform_modifier = obj_to_deform.modifiers['simple_deform']
        deform_modifier.deform_method = 'BEND'

        deform_modifier.deform_axis = 'Z'
        deform_modifier.angle = random.uniform(-0.523, -3.14)

    def adjust_object_position(self, obj, target_size, background_plane):
        """
        moves the objects above the background plane.

        Keyword arguments:
        obj: blender mesh object
        target_size: gap between the object and the background plane.
        """
        max_dim = np.max(obj.dimensions)
        lift_distance = (max_dim + target_size) / 2
        obj.location.z = lift_distance

    def track_camera_object(self, object_to_track):
        camera = bpy.data.objects['Camera']
        if not str('Track To') in camera.constraints.keys():
            camera.constraints.new(type="TRACK_TO")
            camera.constraints['Track To'].target = object_to_track

    def track_light_object(self, object_to_track):
        light = bpy.data.objects['Sun']
        if not str('Track To') in light.constraints.keys():
            light.constraints.new(type="TRACK_TO")
            light.constraints['Track To'].target = object_to_track

    def random_camera_position(self, camera_name):
        camera = bpy.data.objects[str(camera_name)]
        camera.constraints['Follow Path'].offset = random.randint(0, 100)

    def random_light_position(self, light_name):
        light = bpy.data.objects[str(light_name)]
        light.constraints['Follow Path'].offset = random.randint(0, 100)

    def reset_obj_location_rotation(self, obj):
        obj.location = (0.0, 0.0, 0.0)
        obj.rotation_euler = (0.0, 0.0, 0.0)

    def set_camera_postion_on_path(self,camera_name,distance_value):
        camera = bpy.data.objects[str(camera_name)]
        camera.constraints['Follow Path'].offset = distance_value


from collections import deque

N, M = map(int, input().split())
board = [list(map(int, list(input()))) for _ in range(N)]
visited = [[False] * M for _ in range(N)]
dr = [0, 1, 0, -1]
dc = [1, 0, -1, 0]

def bfs():
    q = deque()

    # (row, col, cost)
    q.append((0, 0, 1))
    visited[0][0] = True

    while (q):
        row, col, cost = q.popleft()

        for i in range(len(dr)):
            nr = row + dr[i]
            nc = col + dc[i]

            if (nr < N and nr >= 0 and nc < M and nc >= 0 and board[nr][nc] == 1 and visited[nr][nc] == False):
                visited[nr][nc] = True
                q.append((nr, nc, cost + 1))

                if ((nr, nc) == (N - 1, M - 1)):
                    print(cost + 1)
                    return

bfs()


import argparse
import pickle
import json

parser = argparse.ArgumentParser(
    description='Get neccessary info for picklist and create batches.')
parser.add_argument('input_file', type=str, help='json file')
parser.add_argument('-limit', required=False, type=int, help='integer')
parser.add_argument('-batch_size', required=False, type=int, help='integer')
args = parser.parse_args()

with open(args.input_file, 'r') as data_file:
    json_data = data_file.read()

data = json.loads(json_data)

output_list = []
BATCH_SIZE = 250
file_suffix = 1
if args.batch_size:
    BATCH_SIZE = args.batch_size
if args.limit:
    del data[args.limit:]

for record in data:
    if record['scientific_name'] == 'human gut metagenome':  # picking only fecal samples
        temp_dict = {}
        sample_accession = record['sample_accession']
        temp_dict['sample_accession'] = sample_accession
        temp_dict['run_accession'] = record['run_accession']
        temp_dict['xml_ftp'] = f'https://www.ebi.ac.uk/ena/browser/api/xml/{sample_accession}?download=true'
        temp_dict['fastq_ftp'] = record['fastq_ftp']
        output_list.append(temp_dict)
        if len(output_list) == BATCH_SIZE:  # dividing into batches
            with open(f'manifest_batch{file_suffix}.pickle', 'wb') as filehandle:
                pickle.dump(output_list, filehandle)
                filehandle.close()
            output_list = []
            file_suffix += 1

with open(f'manifest_batch{file_suffix}.pickle', 'wb') as filehandle:
    pickle.dump(output_list, filehandle)
    filehandle.close()


import os
import tkinter as tk
import tkinter.messagebox

import numpy as np
from PIL import ImageTk, Image

from DN_tool.lib.nii2plot_v1 import nii_draw
from DN_tool.lib.nrrd2plot_v1 import nrrd_draw

class draw_contours:
    def __init__(self):
        self.shape = 500
        self.canvas_max = 2000

        self.draw_contour_win = tk.Toplevel()
        self.draw_contour_win.geometry("800x800")
        self.draw_contour_win.title("Draw contous")
        nii_label = tk.Label(self.draw_contour_win,  font=('Times', 18, 'bold'), text="Nii img path:")
        nii_label.place(x=20, y=20, anchor='nw')
        mask_label = tk.Label(self.draw_contour_win, font=('Times', 18, 'bold'), text="Nii/Nrrd mask path:")
        mask_label.place(x=20, y=60, anchor='nw')
        out_label = tk.Label(self.draw_contour_win, font=('Times', 18, 'bold'), text="output path:")
        out_label.place(x=20, y=100, anchor='nw')

        self.niistr = tk.StringVar()
        self.maskstr = tk.StringVar()
        self.outpath = tk.StringVar()
        tk_path = tk.Entry(self.draw_contour_win, textvariable=self.niistr, width=80)
        tk_path.place(x=300, y=30, anchor='nw')
        tk_path = tk.Entry(self.draw_contour_win, textvariable=self.maskstr, width=80)
        tk_path.place(x=300, y=70, anchor='nw')
        tk_path = tk.Entry(self.draw_contour_win, textvariable=self.outpath, width=80)
        tk_path.place(x=300, y=110, anchor='nw')

        tk_btm = tk.Button(self.draw_contour_win, text="生成边缘勾勒", bg='green', fg='red',
                           font=('Helvetica', 20), command=self.draw)
        tk_btm.place(x=400, y=200, anchor='center')
        self.canvas = tk.Canvas(self.draw_contour_win, relief='raised', bg='black', height=800, width=1000)
        # self.canvas.config(scrollregion=self.canvas.bbox("all"))
        self.canvas.config(scrollregion=(0, 0, self.canvas_max, self.canvas_max))
        self.canvas.place(x=10, y=240, anchor='nw')
        self.image_list = []
        self.index = 0
        # scrollbar
        self.vbar = tk.Scrollbar(self.draw_contour_win, orient=tk.VERTICAL, bg='#87CEFA')
        self.vbar.pack(side=tk.RIGHT, fill=tk.Y)
        self.vbar.config(command=self.canvas.yview)
        self.hbar = tk.Scrollbar(self.draw_contour_win, orient=tk.HORIZONTAL, bg='#87CEFA')
        self.hbar.pack(side=tk.BOTTOM, fill=tk.X)
        self.hbar.config(command=self.canvas.xview)
        self.canvas.config(yscrollcommand=self.vbar.set, xscrollcommand=self.hbar.set)

    def draw(self):
        if not os.path.exists(self.niistr.get()):
            tkinter.messagebox.showerror(title='Error', message='Exception happen, Pls check nii path.')
        if not os.path.exists(self.maskstr.get()):
            tkinter.messagebox.showerror(title='Error', message='Exception happen, Pls check mask path.')
        if not os.path.exists(self.outpath.get()):
            tkinter.messagebox.showwarning(title='warning', message='warning, Pls check out path.')
            os.makedirs(self.outpath.get())

        niistr = self.niistr.get()
        maskstr = self.maskstr.get()
        outpath = self.outpath.get()
        if maskstr.endswith(".nrrd"):
            png = nrrd_draw(niistr, maskstr, outpath)
        elif maskstr.endswith(".nii"):
            png = nii_draw(niistr, maskstr, outpath)
        r = Image.fromarray(png[..., 0]).convert("L")
        g = Image.fromarray(png[..., 1]).convert("L")
        b = Image.fromarray(png[..., 2]).convert("L")
        self.image_list.append(ImageTk.PhotoImage(Image.merge("RGB", (r, g, b)).resize((self.shape, self.shape))))
        # calc canvas_locx and canvas_locy
        a, v = np.divmod(self.index, self.canvas_max // self.shape)
        y = a * self.shape
        x = v * self.shape
        self.canvas.create_image(x, y, anchor='nw', image=self.image_list[self.index])
        self.index += 1


import gettext
from gettext import gettext as _

gettext.textdomain('gso')

from gso.__version__  import NAME
from gso.__version__  import DESCRIPTION
from gso.__version__  import URL
from gso.__version__  import AUTHOR
from gso.__version__  import VERSION
from gso.__version__  import TITLE

def _my_gettext(s):
    """Traducir algunas cadenas de argparse."""
    current_dict = {'usage: ': 'uso: ',
                    'optional arguments': 'argumentos opcionales',
                    'show this help message and exit': 'mostrar esta ayuda y salir',
                    'positional arguments': 'argumentos posicionales',
                    'the following arguments are required: %s': 'los siguientes argumentos son requeridos: %s'}

    if s in current_dict:
        return current_dict[s]
    return s

gettext.gettext = _my_gettext

import argparse

def init_argparse():
    """Inicializar parametros del programa."""

    epilog =  """

Patrón de selección:
    <tipo de objeto>.<conexión>.<database>.<owner>.<nombre del objeto>

Tipos de objeto lógico a seleccionar:
    P : Stored procedure
    PV: Stroed procdure (versionado)
    FN: SQL scalar function
    IF: SQL inline table-valued function
    TF: SQL table-valued-function
    FN: Funcion
    IF: Escalr function
    FF: Todas las funciones
    TR: Trigger
    D : Default
    R : Rule
    OP: Operacón
    PR: Parametro
    PA: Procesos Agenda
    TT: Tabla temporal mecanus
    TB: Tabla física de SQL Server
    MO: Modulos
    ME: Menú de sistema
    RP: Reporte

Ejemplos:\n

Para exportar todos los objetos lógicos a un carpeta físicasegún se define en archivo.cfg:
   gso export_db  *.*.*.*.* --config archivo.cfg

Para exportar todos los objetos lógicos a un carpeta física según se define en archivo.cfg:
   gso export_db  *.momdesa1.*.*.* --config archivo.cfg

 Para exportar todos las tablas físicas de cierta base/owner con cierto patron:
   gso export_db  TB.momdesa1.contable_db.dbo.MAC --config archivo.cfg
"""

    cmdparser = argparse.ArgumentParser(prog=NAME,
                                        description=TITLE,
                                        epilog=epilog,
                                        add_help=True,
                                        formatter_class=make_wide(argparse.RawTextHelpFormatter, w=80, h=48)
    )

    opciones = {    "verbo": {
                                "help": _("Accion a ejecutar")
                    },
                    "patron": {
                                "help": _("Patron del objeto a exportar")
                    },
                    "--version -v": {
                                "action":    "version",
                                "version":    VERSION,
                                "help":        _("Mostrar el número de versión y salir")
                    },
                    "--log-level -n": {
                                "type":     str,
                                "action":   "store",
                                "dest":     "loglevel",
                                "default":  "info",
                                "help":     _("Nivel de log")
                    },
                    "--log-file -l": {
                            "type":	str,
                            "action": "store",
                            "dest":	"logfile",
                            "default": "",
                            "help":	_("Archivo de log"),
                            "metavar": "file"
                    },
                    "--config-file -c": {
                            "type":	str,
                            "action": "store",
                            "dest":	"config_file",
                            "default": "gso.cfg",
                            "help":	_("Archivo de configuración"),
                            "metavar": "file"
                    },
                    "--quiet -q": {
                                "action":     "store_true",
                                "dest":     "quiet",
                                "default":    False,
                                "help":        _("Modo silencioso sin mostrar absolutamente nada.")
                    },
                    "--last-n-days -d": {
                            "type":	int,
                            "action": "store",
                            "dest":	"ndays",
                            "default": None,
                            "help":	_("Configura la cantidad de días de los objetos modificados a recuperar"),
                            "metavar": "dias"
                    },
            }

    for key, val in opciones.items():
        args = key.split()
        kwargs = {}
        kwargs.update(val)
        cmdparser.add_argument(*args, **kwargs)

    return cmdparser

def make_wide(formatter, w=120, h=40):
    """Return a wider HelpFormatter, if possible."""
    try:
        # https://stackoverflow.com/a/5464440
        # beware: "Only the name of this class is considered a public API."
        kwargs = {'width': w, 'max_help_position': h}
        formatter(None, **kwargs)
        return lambda prog: formatter(prog, **kwargs)
    except TypeError:
        warnings.warn("argparse help formatter failed, falling back.")
        return formatter


import requests
import pygal
from pygal.style import LightColorizedStyle as LCS, LightenStyle as LS

url = 'https://api.github.com/search/repositories?q=language:Java&sort=stars'
r = requests.get(url)
print("Status code:", r.status_code)

response_dict = r.json()
print("Total repositories:", response_dict['total_count'])

repo_dicts = response_dict['items']

my_style = LS('#333366', base_style=LCS)
my_style.tittle_font_size = 24
my_style.label_font_size = 14
my_style.major_label_font_size = 18
my_config = pygal.Config()
my_config.x_label_rotation = 45
my_config.show_legend = False
my_config.truncate_label = 15
my_config.show_y_guides = False
my_config.width = 1000

chart = pygal.Bar(my_config, style=my_style)
chart.title = 'Most-Starred JavaScript Projects on GitHub'

chart_data = []
names = []
for repo_dict in repo_dicts:
    repo_chart_data = {
        'value': repo_dict['stargazers_count'],
        'label': repo_dict['name'],
        'xlink': repo_dict['html_url']
    }
    chart_data.append(repo_chart_data)
    names.append(repo_dict['name'])

chart.add('Some value', chart_data)
chart.x_labels = names
chart.render_to_file('java_repos.svg')


import os
import datetime as dt
import json
from subprocess import check_call, check_output, CalledProcessError

import pandas as pd
import requests
from dotenv import load_dotenv

from .utilities import update_hourly

ROOT = os.path.dirname(os.path.abspath(__file__))
RAW_DATA_PATH = os.path.join(ROOT, "..", "data", "raw", "co2signal.com")
CLEAN_DATA_PATH = os.path.join(ROOT, "..", "data", "clean", "co2signal.com")

def co2signal_get_latest(token: str, country_code: str="CA-ON") -> dict:
    url = f"https://api.co2signal.com/v1/latest?countryCode={country_code}"
    response = requests.get(url, headers={'auth-token': f'{token}'})
    if response.status_code != 200:
        raise RuntimeError(response.status_code)
    json_string = json.dumps(json.loads(response.content), indent=4)
    print(json_string)
    with open(os.path.join(RAW_DATA_PATH, country_code, "latest", "output.json"), "w") as f:
        f.write(json_string)
    return json.loads(json_string)

def clean_json(json_output: dict) -> dict:
    json_output.pop("_disclaimer")
    json_output.pop("status")
    json_output.pop("countryCode")
    json_output["datetime"] = pd.to_datetime(
        json_output["data"]["datetime"]
    ).tz_convert("America/Toronto").isoformat()
    json_output["data"].pop("datetime")
    return json_output

def main():
    load_dotenv()
    print("query co2signal...")
    if "CO2SIGNAL_API_TOKEN" in os.environ.keys():
        json_output = co2signal_get_latest(os.environ["CO2SIGNAL_API_TOKEN"])
        clean_json(json_output)
        with open(os.path.join(CLEAN_DATA_PATH, "CA-ON", "latest", "output.json"), "w") as f:
            f.write(json.dumps(json_output, indent=4))

        # Commit changes
        check_call(["git", "add", "data"])
        try:
            check_output(['git', 'commit', '-m', 'update data'])
        except CalledProcessError as e:
            if (
                'no changes added to commit' not in e.output.decode("utf-8") and
                'nothing to commit' not in e.output.decode("utf-8")
             ):
                print(e.output)
                raise

    filepath = os.path.abspath(os.path.join(CLEAN_DATA_PATH, "CA-ON", "latest", "output.json"))
    hourly_path = os.path.abspath(os.path.join(CLEAN_DATA_PATH, "CA-ON", "hourly", "output.csv"))
    update_hourly(filepath, hourly_path)

if __name__ == "__main__":
    main()


import collections

# Returns a float between 0 and 1 representing how
# similar two documents are to each other
def get_similarity(doc1, doc2):
    word_match_occurrence = 0

    doc1_word_map, doc1_word_count = get_word_map(doc1)
    doc2_word_map, doc2_word_count = get_word_map(doc2)

    # Iterate through each key value pair of doc1 word map
    # and check if the key is doc2 word map
    for key, value in doc1_word_map.items():

        # If the value is present then add twice the lowest value to the word match occurrence counter.
        # (This is the total number of times that word appears in BOTH bodies of text)
        if key in doc2_word_map:
            word_match_occurrence += min(value, doc2_word_map[key]) * 2

    # Return the ratio between the occurrence of the word matches and the count of all words
    return word_match_occurrence / (doc1_word_count + doc2_word_count)

# Returns a map and count of words appearing in a given document
def get_word_map(doc):
    word_map = collections.defaultdict(int)
    word_count = 0

    with open(doc, 'r') as f:
        for line in f:
            for word in line.split():
                word_map[word.lower()] += 1
                word_count += 1

    return word_map, word_count

print(get_similarity('text1.txt', 'text2.txt'))


import pytest
import shilofue.Parse as Parse

def test_build_group_from_dict():
    # test 0-1, raise error because input type is not dict
    with pytest.raises(TypeError) as excinfo:
        _names, _parameters = Parse.GetGroupCaseFromDict('foo')
    assert('dictionary' in str(excinfo.value))
    # test 0-2, raise error because some entry is not int, float or str
    _idict = {}
    _idict['test parameter'] = None
    with pytest.raises(TypeError) as excinfo:
        _names, _parameters = Parse.GetGroupCaseFromDict(_idict)
    assert('int, float or str' in str(excinfo.value))
    # test 0-3
    # todo error in ExpandNamesParameters function

    # test 1, corresponds to
    # 'set Test parameter = 0 or 1 or 2'
    # in a .prm file
    _idict={}
    _idict['test parameter'] = [0, 1, 2]
    _names, _parameters = Parse.GetGroupCaseFromDict(_idict)
    assert(_names == [['test parameter']])
    assert(_parameters == [[0, 1, 2]])
    _cases_config = Parse.ExpandNamesParameters(_names, _parameters)
    # _cases_config contains 3 cases, which is a list of 3
    # each case has a dictionary with two keys: 'names', 'values'
    assert(len(_cases_config) == 3)
    assert(_cases_config[0]['names'] == [['test parameter']])
    assert(_cases_config[1]['names'] == [['test parameter']])
    assert(_cases_config[2]['names'] == [['test parameter']])
    assert(_cases_config[0]['values'] == [0])
    assert(_cases_config[1]['values'] == [1])
    assert(_cases_config[2]['values'] == [2])
    # test 2, correspond to
    #   'Subsection Material model
    #       subsection Visco Plastic
    #           set Reference viscosity = 1e20 or 1e21 or 1e22
    #           set Maximum viscosity = 1e24
    #
    #   '
    # in a .prm file
    _idict={}
    _idict['Start time'] = [0, 1e6]
    _idict['Material model'] = {'Visco Plastic': {'Reference viscosity': [1e20, 1e21, 1e22], 'Maximum viscosity': [1e24, 1e25]}}
    _names, _parameters = Parse.GetGroupCaseFromDict(_idict)
    assert(_names == [['Material model', 'Visco Plastic', 'Maximum viscosity'], ['Material model', 'Visco Plastic', 'Reference viscosity'] ,['Start time']])
    assert(_parameters == [[1e24, 1e25], [1e20, 1e21, 1e22], [0, 1e6]])
    _cases_config = Parse.ExpandNamesParameters(_names, _parameters)
    assert(len(_cases_config) == 12)
    for i in range(12):
        assert(_cases_config[i]['names'] == _names)
    assert (_cases_config[0]['values'] == [1e24, 1e20, 0])
    assert (_cases_config[1]['values'] == [1e24, 1e20, 1e6])
    assert (_cases_config[11]['values'] == [1e25, 1e22, 1e6])

def test_get_group_case_from_dict1():
    '''
    Test the function GetGroupCaseFromDict1
    '''
    # test1
    # test dictionary
    _idict = {'config': {'foo': [0, 1, 2]},
              'test': {'foo1': [1, 2, 3]}}
    # call function
    _config_tests = Parse.GetGroupCaseFromDict1(_idict)
    # assertion
    _standard_config_tests = [
                              {'config': {'foo': 0}, 'test': {'foo1': 1}},
                              {'config': {'foo': 1}, 'test': {'foo1': 1}},
                              {'config': {'foo': 2}, 'test': {'foo1': 1}},
                              {'config': {'foo': 0}, 'test': {'foo1': 2}},
                              {'config': {'foo': 1}, 'test': {'foo1': 2}},
                              {'config': {'foo': 2}, 'test': {'foo1': 2}},
                              {'config': {'foo': 0}, 'test': {'foo1': 3}},
                              {'config': {'foo': 1}, 'test': {'foo1': 3}},
                              {'config': {'foo': 2}, 'test': {'foo1': 3}}
                             ]
    assert(_config_tests == _standard_config_tests)
    # test2
    # test dictionary
    _idict = {'config': {'foo': [0, 1], 'foo1': [0, 1]},
              'test': {'foo2': [0, 1]}}
    # call function
    _config_tests = Parse.GetGroupCaseFromDict1(_idict)
    # assertion
    _standard_config_tests = [
                              {'config': {'foo': 0, 'foo1': 0}, 'test': {'foo2': 0}},
                              {'config': {'foo': 1, 'foo1': 0}, 'test': {'foo2': 0}},
                              {'config': {'foo': 0, 'foo1': 1}, 'test': {'foo2': 0}},
                              {'config': {'foo': 1, 'foo1': 1}, 'test': {'foo2': 0}},
                              {'config': {'foo': 0, 'foo1': 0}, 'test': {'foo2': 1}},
                              {'config': {'foo': 1, 'foo1': 0}, 'test': {'foo2': 1}},
                              {'config': {'foo': 0, 'foo1': 1}, 'test': {'foo2': 1}},
                              {'config': {'foo': 1, 'foo1': 1}, 'test': {'foo2': 1}}
    ]
    assert(_config_tests == _standard_config_tests)

    # test 3: include sub_group options
    # _idict = {'config': {'foo': [0, 1], 'foo1': [0, 1], 'sub_group1': {['foo2', 'foo3']: [[2, 3], [4, 5]]}},
    #          'test': {'foo2': [0, 1]}}

def test_change_disc_values():
    '''
    Test the function ChangeDiscValues()
    '''
    # test 1
    # initiate the dictionary
    _idict={}
    _idict['Start time'] = 0
    _idict['Material model'] = {'Visco Plastic': {'Reference viscosity': 1e20, 'Maximum viscosity': 1e24}}
    # initiate two lists
    _names = [['Material model', 'Visco Plastic', 'Maximum viscosity'], ['Material model', 'Visco Plastic', 'Reference viscosity'] ,['Start time']]
    _values = [1e25, 1e22, 1e6]
    Parse.ChangeDiscValues(_idict, _names, _values)  # change value within _idict
    assert(_idict['Start time'] == 1e6)
    assert(_idict['Material model'] == {'Visco Plastic': {'Reference viscosity': 1e22, 'Maximum viscosity': 1e25}})

def test_pattern_from_str():
    '''
    Test the function PatterFromStr
    '''
    assert(Parse.PatternFromStr('foo_foo') == 'FF')
    assert(Parse.PatternFromStr('upper_lower_viscosity') == 'ULV')
    pass

def test_pattern_from_value():
    '''
    Test the function PatterFromValue
    '''
    assert(Parse.PatternFromValue('spherical') == 's')
    assert(Parse.PatternFromValue(0.1) == '1.000e-01')
    assert(Parse.PatternFromValue(100) == '100')
    pass


from django.core.exceptions import ObjectDoesNotExist
from django.http.response import Http404
from django.shortcuts import render
from .models import Project

def index(request):
    projects = Project.objects.all()
    context = {'projects': projects}
    return render(request, 'projects/index.html', context)

def detail_slug(request, slug):
    try:
        project = Project.objects.get(slug=slug)
    except ObjectDoesNotExist:
        raise Http404
    context = {'project': project}
    return render(request, 'projects/detail.html', context)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('users_b', '0003_auto_20160101_2355'),
        ('search', '0004_book_ean_13'),
    ]

    operations = [
        migrations.CreateModel(
            name='StaredBook',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('book', models.ForeignKey(to='search.Book')),
                ('user', models.ForeignKey(to='users_b.User')),
            ],
            options={
                'ordering': ['-created_at'],
            },
        ),
    ]


def g(i,j):
    try:
        import urllib.request as r
        page_n=i*44
        city_l=('%E5%8C%97%E4%BA%AC','%E4%B8%8A%E6%B5%B7','%E5%B9%BF%E5%B7%9E','%E6%B7%B1%E5%9C%B3','%E6%9D%AD%E5%B7%9E','%E9%95%BF%E6%B2%99','%E9%95%BF%E6%98%A5','%E6%88%90%E9%83%BD','%E9%87%8D%E5%BA%86','%E5%A4%A7%E8%BF%9E')
        city=city_l[j]#'%E5%8C%97%E4%BA%AC'
        page='&bcoffset=4&ntoffset=4&p4ppushleft=1%2C48&loc={}&s={}'.format(city,page_n)
        url='https://s.taobao.com/search?q=%E7%94%B5%E8%84%91&imgfile=&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=a21bo.2017.201856-taobao-item.1&ie=utf8&initiative_id=tbindexz_20170306'+page+'&ajax=true'
        data=r.urlopen(url).read().decode('utf-8','ignore')

        import json
        data=json.loads(data)
        a=json.dumps(data)

        #def info(x):
            #return data['mods']['itemlist']['data']['auctions'][x]
        #def loc(x):
            #return data['mods']['itemlist']['data']['auctions'][x]['item_loc']
        #def name(x):
            #return data['mods']['itemlist']['data']['auctions'][x]['raw_title']
        def price(x):
            return data['mods']['itemlist']['data']['auctions'][x]['view_price']
        def sales(x):
            return data['mods']['itemlist']['data']['auctions'][x]['view_sales']
        print('第{}页商品信息：\n'.format(i+1))
        f=open('北京到大连价格.txt','a',encoding='utf-8')
        f.write(a+'\n')
        f.close()
    except Exception as err:
        print('打印失败')
for j in range(10):
    print('打印第{}个城市中'.format(j+1))
    for i in range(100):
        g(i,j)


import logging
import re
from collections import Counter, OrderedDict

import espresso.tools.utils as speech_utils
from fairseq.file_io import PathManager

logger = logging.getLogger(__name__)

class Scorer(object):
    def __init__(self, dictionary, wer_output_filter=None):
        self.dictionary = dictionary
        self.ordered_utt_list = None
        self.word_filters = []
        self.parse_wer_output_filter(wer_output_filter)
        self.reset()

    def reset(self):
        self.char_counter = Counter()
        self.word_counter = Counter()
        self.char_results = OrderedDict()
        self.results = OrderedDict()
        self.aligned_results = OrderedDict()

    def parse_wer_output_filter(self, wer_output_filter):
        if wer_output_filter:
            with open(
                PathManager.get_local_path(wer_output_filter), "r", encoding="utf-8"
            ) as f:
                for line in f:
                    line = line.strip()
                    if line.startswith("#!") or line == "":
                        continue
                    elif line.startswith("s/"):
                        m = re.match(r"s/(.+)/(.*)/g", line)
                        assert m is not None
                        self.word_filters.append([m.group(1), m.group(2)])
                    elif line.startswith("s:"):
                        m = re.match(r"s:(.+):(.*):g", line)
                        assert m is not None
                        self.word_filters.append([m.group(1), m.group(2)])
                    else:
                        logger.warning(
                            "Unsupported pattern: '{}'. Ignoring it".format(line)
                        )

    def add_prediction(self, utt_id, pred):
        if not isinstance(utt_id, str):
            raise TypeError("utt_id must be a string(got {})".format(type(utt_id)))
        if not isinstance(pred, str):
            raise TypeError("pred must be a string(got {})".format(type(pred)))

        assert (
            utt_id not in self.char_results
        ), "Duplicated utterance id detected: {}".format(utt_id)
        self.char_results[utt_id] = pred + "\n"

        pred_words = self.dictionary.wordpiece_decode(pred)
        assert (
            utt_id not in self.results
        ), "Duplicated utterance id detected: {}".format(utt_id)
        self.results[utt_id] = pred_words + "\n"

    def add_evaluation(self, utt_id, ref, pred):
        if not isinstance(utt_id, str):
            raise TypeError("utt_id must be a string(got {})".format(type(utt_id)))
        if not isinstance(ref, str):
            raise TypeError("ref must be a string (got {})".format(type(ref)))
        if not isinstance(pred, str):
            raise TypeError("pred must be a string(got {})".format(type(pred)))

        # filter out any non_lang_syms from ref and pred
        non_lang_syms = getattr(self.dictionary, "non_lang_syms", None)
        assert non_lang_syms is None or isinstance(non_lang_syms, list)
        if non_lang_syms is not None and len(non_lang_syms) > 0:
            ref_list, pred_list = ref.strip().split(), pred.strip().split()
            ref = " ".join([x for x in ref_list if x not in non_lang_syms])
            pred = " ".join([x for x in pred_list if x not in non_lang_syms])

        # char level counts
        _, _, counter = speech_utils.edit_distance(
            ref.strip().split(),
            pred.strip().split(),
        )
        self.char_counter += counter

        # word level counts
        ref_words = self.dictionary.wordpiece_decode(ref)
        pred_words = self.dictionary.wordpiece_decode(pred)

        # filter words according to self.word_filters (support re.sub only)
        for pattern, repl in self.word_filters:
            ref_words = re.sub(pattern, repl, ref_words)
            pred_words = re.sub(pattern, repl, pred_words)

        ref_word_list, pred_word_list = ref_words.split(), pred_words.split()
        _, steps, counter = speech_utils.edit_distance(
            ref_word_list,
            pred_word_list,
        )
        self.word_counter += counter
        assert (
            utt_id not in self.aligned_results
        ), "Duplicated utterance id detected: {}".format(utt_id)
        self.aligned_results[utt_id] = speech_utils.aligned_print(
            ref_word_list,
            pred_word_list,
            steps,
        )

    def cer(self):
        assert self.char_counter["words"] > 0
        cer = (
            float(
                self.char_counter["sub"]
                + self.char_counter["ins"]
                + self.char_counter["del"]
            )
            / self.char_counter["words"]
            * 100
        )
        sub = float(self.char_counter["sub"]) / self.char_counter["words"] * 100
        ins = float(self.char_counter["ins"]) / self.char_counter["words"] * 100
        dlt = float(self.char_counter["del"]) / self.char_counter["words"] * 100
        return cer, sub, ins, dlt

    def wer(self):
        assert self.word_counter["words"] > 0
        wer = (
            float(
                self.word_counter["sub"]
                + self.word_counter["ins"]
                + self.word_counter["del"]
            )
            / self.word_counter["words"]
            * 100
        )
        sub = float(self.word_counter["sub"]) / self.word_counter["words"] * 100
        ins = float(self.word_counter["ins"]) / self.word_counter["words"] * 100
        dlt = float(self.word_counter["del"]) / self.word_counter["words"] * 100
        return wer, sub, ins, dlt

    def tot_word_error(self):
        return (
            self.word_counter["sub"]
            + self.word_counter["ins"]
            + self.word_counter["del"]
        )

    def tot_word_count(self):
        return self.word_counter["words"]

    def tot_char_error(self):
        return (
            self.char_counter["sub"]
            + self.char_counter["ins"]
            + self.char_counter["del"]
        )

    def tot_char_count(self):
        return self.char_counter["words"]

    def add_ordered_utt_list(self, *args):
        if len(args) == 1 and isinstance(
            args[0], list
        ):  # aleady a list of utterance ids
            self.ordered_utt_list = args[0]
            return
        self.ordered_utt_list = []
        for text_file in args:
            with open(
                PathManager.get_local_path(text_file), "r", encoding="utf-8"
            ) as f:
                one_utt_list = [line.strip().split()[0] for line in f]
                self.ordered_utt_list.extend(one_utt_list)
        if len(self.char_results):
            assert set(self.ordered_utt_list) == set(self.char_results.keys())
        if len(self.results):
            assert set(self.ordered_utt_list) == set(self.results.keys())
        if len(self.aligned_results):
            assert set(self.ordered_utt_list) == set(self.aligned_results.keys())

    def print_char_results(self):
        res = ""
        if self.ordered_utt_list is not None:
            assert set(self.ordered_utt_list) == set(self.char_results.keys())
            for utt_id in self.ordered_utt_list:
                res += utt_id + " " + self.char_results[utt_id]
        else:
            for utt_id in self.char_results:
                res += utt_id + " " + self.char_results[utt_id]
        return res

    def print_results(self):
        res = ""
        if self.ordered_utt_list is not None:
            assert set(self.ordered_utt_list) == set(self.results.keys())
            for utt_id in self.ordered_utt_list:
                res += utt_id + " " + self.results[utt_id]
        else:
            for utt_id in self.results:
                res += utt_id + " " + self.results[utt_id]
        return res

    def print_aligned_results(self):
        res = ""
        if self.ordered_utt_list is not None:
            assert set(self.ordered_utt_list) == set(self.aligned_results.keys())
            for utt_id in self.ordered_utt_list:
                res += utt_id + "\n" + self.aligned_results[utt_id]
        else:
            for utt_id in self.aligned_results:
                res += utt_id + "\n" + self.aligned_results[utt_id]
        return res