## @package Read raklida datasets
import sys
import os
import h5py
import pandas as pd
import numpy as np
import rasterio

class raklida_import:
    def __init__(self, args):
        """ configure output """
        ## command line arguments
        self.args = args
        ## configuration of parameter properties
        self.conf = {
            'raklida_messungen' : {
                'parameters' : {
                   'ETP' : {'id':8, 'decimals':2, 'dtype':'int32', 'to_hdf' : True},
                   'GRV' : {'id':30, 'decimals':2, 'dtype':'int32', 'to_hdf' : True}
                }
            },
            'raklida_referenz' : {
                 'parameters' : {
                   'RRK' :{'id':10, 'decimals':2, 'dtype':'int32', 'to_hdf' : True},
                   'TM0' : {'id':9, 'decimals':2, 'dtype':'int32', 'to_hdf' : True}
                }
            },
            'raklida_wettreg66' : {
                 'parameters' : {
                   'ETP' :{'id':8, 'decimals':2, 'dtype':'int32', 'to_hdf' : True},
                   'RRK' : {'id':10, 'decimals':2, 'dtype':'int32', 'to_hdf' : True},
                   'TM0' : {'id':9, 'decimals':2, 'dtype':'int32', 'to_hdf' : True}
                }
            },
            'raklida_referenz_derived' : {
                'parameters' : {
                    'pi_summer' : {'id':6, 'decimals':2, 'dtype':'int32', 'to_hdf' : True},
                    'pi_winter' : {'id':7, 'decimals':2, 'dtype':'int32', 'to_hdf' : True}
                }
            },
            'raklida_wettreg66_derived' : {
                'parameters' : {
                    'pi_summer' : {'id':6, 'decimals':2, 'dtype':'int32', 'to_hdf' : True},
                    'pi_winter' : {'id':7, 'decimals':2, 'dtype':'int32', 'to_hdf' : True}
                }
            }
        }

    ## @brief Read asc files
    #
    # @param src, string : source folder
    # @return dbf_list, list: list of files
    def read_asc(self, src):
        print('--> read raklida asc files')
        files = []

        # get raklida asc files
        if 'Raklida' in src:
            for r, d, f in os.walk(src):
                for file in f:
                    if '.asc' in file:
                        if not 'lock' in file: # exlude locked files
                            if not '.txt' in file:
                                files.append(os.path.join(r, file))

        if len(files) > 0:
            print('OK, found %d asc files' % len(files))
            #for i in files:
            #    print(i)
        else:
            sys.exit('ERROR: No asc files found')

        return files

    ## @brief read asc into pandas dataframe
    #
    # @param args, dict: dictionary of arguments
    # @returns stacked_df: pandas dataframe
    def read_as_pandas(self, args):
        print('--> read asc into pandas dataframe')
        # placeholder
        df_all = {}
        u_pnames = []
        # read raklida area dataset
        try:
            raklida_area = h5py.File(args['src_folder'] + '/raklida_gis/raklida.h5', 'r')
            raklida_ds = np.flipud(np.array(raklida_area['Band1'])).flatten().astype(float)
            raklida_ds[raklida_ds<0] = np.nan
        except:
            sys.exit("ERROR: " + str(sys.exc_info()))
        # get unique parameters
        for f in args['asc_files']:
            pname, idparam = self.get_parameter_from_fname(args, f)
            if not pname in u_pnames:
                u_pnames.append(pname)
        print('>>>>> parameters', u_pnames)

        # loop over parameter names
        for pname in u_pnames:
            #if pname == 'TM0':
            # create a key in dict if not exists
            if not pname in df_all.keys():
                df_all[pname] = pd.DataFrame()
            df_raklida = pd.DataFrame()
            next_year = 2100
            # initialize df_stacked
            df_stacked = pd.DataFrame({'id':raklida_ds})
            df_stacked = df_stacked[df_stacked['id'].notna()]
            # read csv file
            args['asc_files'] = sorted(args['asc_files'])
            #print('--> asc_files: ', args)

            for f in args['asc_files']:
                if not '.xml' in f:
                    if pname in f:
                        print('reading by rasterio', f)
                        # get year and month from filename
                        year, month = self.get_year_month_from_fname(f)

                        print(year, next_year)

                        #handle missing years
                        if next_year < year:
                            next_year = year + 1
                            # reinitialize df_stacked
                            df_stacked = pd.DataFrame({'id':raklida_ds})
                            df_stacked = df_stacked[df_stacked['id'].notna()]
                            print('*****************', str(year))
                            print(f)

                        raster = rasterio.open(f).read(1).flatten() # read file by rasterio
                        # get nodata value
                        nodata = self.get_nodata_value_from_asc(f)
                        raster[raster == nodata] = np.nan
                        # loop over one year
                        if year <= next_year:
                            if year != next_year and month != 'dec':
                                # stack data into pandas dataframe
                                print(raklida_ds.shape)
                                print(raster.shape)
                                print(month)

                                df = pd.DataFrame({'id':raklida_ds, 'value':raster}).rename(columns={'value':month})
                                df = df[df['id'].notna()]
                                df_stacked['year'] = year
                                df_stacked = pd.merge(df_stacked, df, how='left', on='id')
                                print(df_stacked.shape)
                            elif year != next_year and  month == 'dec':
                                # stack data of one year into pandas dataframe
                                df = pd.DataFrame({'id':raklida_ds, 'value':raster}).rename(columns={'value':month})
                                df = df[df['id'].notna()]
                                df_stacked['year'] = year
                                df_stacked = pd.merge(df_stacked, df, how='left', on='id')
                                # add df_stacked (one complete year) to df_all
                                frames = [df_raklida, df_stacked]
                                df_raklida = pd.concat(frames)
                                print(df_raklida.shape, df_stacked.shape)
                                print('df_raklida.shape: ', df_raklida.shape)
                                #sys.exit()
                                if df_raklida.shape[1] > 14:
                                    print(df_stacked)
                                    sys.exit('ERROR: shape not correct')
                            else:
                                # reinitialize df_stacked
                                df_stacked = pd.DataFrame({'id':raklida_ds})
                                df_stacked = df_stacked[df_stacked['id'].notna()]
                                df = pd.DataFrame({'id':raklida_ds, 'value':raster}).rename(columns={'value':month})
                                df = df[df['id'].notna()]
                                df_stacked['year'] = year
                                df_stacked = pd.merge(df_stacked, df, how='left', on='id')
                                print('##### ', str(year))
                                #break
                                #print(df_stacked)
                            next_year = year + 1

                        #if next_year == 2023:
                        #    break

                    #break
                df_all[pname] = df_raklida
                #break
                #print(df_all)
                #sys.exit()
        return df_all

    ## @brief Get nodata value
    #
    # @param file, string
    # @returns nodata, float
    def get_nodata_value_from_asc(self, file):
        print('--> get nodata value from file')
        nodata = None
        file = open(file, 'r')
        for line in file.readlines():
            if 'nodata_value' in line:
                nodata = float((line.split(' '))[2])
                break
        file.close()
        if nodata == None:
            sys.exit('ERROR: Missing nodata value in asc file', file)

        return nodata

    ## @brief Get year and month from filename
    #
    # @param filename: string
    # @returns year, month: integer
    def get_year_month_from_fname(self, file):
        print('--> get year and month from filename')
        year, month = None, None
        months = {1:'jan', 2:'feb', 3:'mar', 4:'apr', 5:'may', 6:'jun', 7:'jul', 8:'aug', 9:'sep', 10:'oct', 11:'nov', 12:'dec'}

        yearmonth = file.split('.')[0].split('_')[-1]
        year = int(yearmonth[0:4])

        if int(yearmonth[4:6]) in range(1,13):
            month = int(yearmonth[4:6])

        if year == None or month == None:
            sys.exit('ERROR: Missing year or month')

        return year, months[month]

    ## @brief Get parameter name and id from filename
    #
    # @param filename: string
    # @returns pname, id: integer
    def get_parameter_from_fname(self, args, file):

        #print ('---> : file :',file)
        #print ('---> : conf :',str(self.conf[args['data_type']]['parameters']))

        #print('--> get parameter name and id from filename')
        pname, idparam = None, None
        for p in self.conf[args['data_type']]['parameters']:
            if p in file:
                idparam = self.conf[args['data_type']]['parameters'][p]['id']
                pname = p

        if pname == None or idparam == None:
            sys.exit('ERROR: Missing parameter name or parameter id')

        return pname, idparam

    ## @brief Save parameters as hdf5
    #
    # @param df, pandas DataFrame
    def transform_as_hdf(self, args, dict):
        print('--> transform data to hdf5')
        print(dict.keys())
        # loop over each parameter
        for p in dict.keys():
            if self.conf[args['data_type']]['parameters'][p]['to_hdf'] == True:
                # get dataset for a parameter
                df = dict[p]
                print('parameter --> ', p)
                start_year = df['year'].min()
                last_year = df['year'].max()
                print('start year', start_year)
                print('last year', last_year)
                for y in range(int(start_year),int(last_year)+1):
                    print(y)
                    # get data for one year
                    df_year = df.loc[df['year'] == y]
                    # reorganize data to final structure: index, values_block_0[]
                    df_final = pd.DataFrame(columns=['index','values_block_0'])
                    df_final['index'] = df_year['id']
                    df_final['values_block_0'] = df_year[df_year.columns[2:]].apply(
                        lambda x: ' '.join(x.astype(str)),
                        axis=1
                    )
                    df_final = df_final.set_index('index')

                    print(df_final.dtypes)
                    # save as hdf5
                    path = self.args['dst_folder'] + '/' + str(self.args['scenario_id']) + '/' + str(self.conf[args['data_type']]['parameters'][p]['id']) + '/'
                    if not os.path.exists(path):
                        os.makedirs(path)
                    fn = str(self.args['scenario_id']) + '_' + str(self.conf[args['data_type']]['parameters'][p]['id']) + '_' + str(y) + '.h5'
                    print(path + fn)
                    #sys.exit()
                    try:
                        df_final[['values_block_0']].to_hdf(path + fn, key='table' , mode='w', format='table', complevel=9)
                    except:
                        sys.exit("ERROR: " + str(sys.exc_info()))

    ## @brief create derived parameters and store them into a pandas dataframe
    #
    # @param args, dict: dictionary of arguments
    # @returns stacked_df: pandas dataframe
    def create_derived_parameters(self, args):
        print('--> Create derived parameters')
        df = {}
        # loop over derived parameters to create
        for p in self.conf[args['data_type']]['parameters']:
            if p == 'pi_summer' and self.conf[args['data_type']]['parameters'][p]['to_hdf'] == True:
                print('+++')
                df['pi_summer'] = self.derive_pi_summer(args)

            if p == 'pi_winter' and self.conf[args['data_type']]['parameters'][p]['to_hdf'] == True:
                print('+++')
                df['pi_winter'] = self.derive_pi_winter(args)

        if len(df) == 0:
            sys.exit('ERROR: dataframe creation failed')

        return df

    ## @brief Derive pi_summer
    #
    # sum of parameters 12, 13, 25, 26, 27, 28, 29
    def derive_pi_summer(self, args):
        print('--> Derive pi_summer')
        ds = {}
        # loop over files
        src = args['src_folder'] + '/' + str(args['scenario_id']) + '/10/'
        print('--> src', src)
        for r, d, f in os.walk(src):
            for file in f:
                df = pd.DataFrame()
                # get year from fname
                year = file.split('.')[0].split('_')[-1]
                print(year)

                try:
                     # read parameters for one year# read the h5 file
                    df_10 = pd.read_hdf(args['src_folder'] + '/' + str(args['scenario_id']) + '/10/' + str(args['scenario_id']) + '_10_' + str(year) + '.h5', key='table').reset_index()
                    # set integer type for column, set nan values
                    df_10['index'] = df_10['index'].astype('float').replace(-99999, np.nan)
                    # get a column for each month
                    df_10[['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']] = df_10.values_block_0.str.split(expand=True,).astype(float).replace(-99999, np.nan)
                    # remove column values_block_0
                    df_10 = df_10.drop(columns=['values_block_0'])
                    # create sum of flows
                    decimals = self.conf[args['data_type']]['parameters']['pi_summer']['decimals']
                    df['index'] = df_10['index']
                    df['jan'] = 0
                    df['feb'] = 0
                    df['mar'] = 0
                    df['apr'] = df_10['apr'].round(decimals)
                    df['may'] = df_10['may'].round(decimals)
                    df['jun'] = df_10['jun'].round(decimals)
                    df['jul'] = df_10['jul'].round(decimals)
                    df['aug'] = df_10['aug'].round(decimals)
                    df['sep'] = df_10['sep'].round(decimals)
                    df['oct'] = 0
                    df['nov'] = 0
                    df['dec'] = 0
                    ds[str(year)] = df
                except:
                    print("ERROR: ")

        return ds

    ## @brief Derive pi_winter
    #
    # sum of parameters 12, 13, 25, 26, 27, 28, 29
    def derive_pi_winter(self, args):
        print('--> Derive pi_winter')
        ds = {}
        # loop over files
        src = args['src_folder'] + '/' + str(args['scenario_id']) + '/10/'
        print('--> src', src)
        for r, d, f in os.walk(src):
            for file in f:
                df = pd.DataFrame()
                # get year from fname
                year = file.split('.')[0].split('_')[-1]
                print(year)

                try:
                    # read parameters for one year# read the h5 file
                    df_10 = pd.read_hdf(args['src_folder'] + '/' + str(args['scenario_id']) + '/10/' + str(args['scenario_id']) + '_10_' + str(year) + '.h5', key='table').reset_index()
                    # set integer type for column, set nan values
                    df_10['index'] = df_10['index'].astype('float').replace(-99999, np.nan)
                    # get a column for each month
                    df_10[['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']] = df_10.values_block_0.str.split(expand=True,).astype(float).replace(-99999, np.nan)
                    # remove column values_block_0
                    df_10 = df_10.drop(columns=['values_block_0'])
                    # create sum of flows
                    decimals = self.conf[args['data_type']]['parameters']['pi_winter']['decimals']
                    df['index'] = df_10['index']
                    df['jan'] = df_10['jan'].round(decimals)
                    df['feb'] = df_10['feb'].round(decimals)
                    df['mar'] = df_10['mar'].round(decimals)
                    df['apr'] = 0
                    df['may'] = 0
                    df['jun'] = 0
                    df['jul'] = 0
                    df['aug'] = 0
                    df['sep'] = 0
                    df['oct'] = df_10['oct'].round(decimals)
                    df['nov'] = df_10['nov'].round(decimals)
                    df['dec'] = df_10['dec'].round(decimals)
                    # assign dataframe to dictionary
                    ds[str(year)] = df
                except:
                    print("ERROR: ")

        return ds

    ## @brief Save derived parameters
    def save_derived_parameters(self, args, df):
        print('--> Save derived parameters')
        for p in df:
            print('===>>', p)
            if df[p] != None:
                for y in df[p]:
                    print(y)
                    df_year = df[p][str(y)]
                    # reorganize data to final structure: index, values_block_0[]
                    df_final = pd.DataFrame(columns=['index','values_block_0'])
                    df_final['index'] = df_year['index']
                    df_final['values_block_0'] = df_year[df_year.columns[1:]].apply(
                        lambda x: ' '.join(x.astype(str)),
                        axis=1
                    )
                    df_final = df_final.set_index('index')
                    #print(df_final)
                    print(df_final.dtypes)
                    # save as hdf5
                    path = self.args['dst_folder'] + str(self.args['scenario_id']) + '/' + str(self.conf[args['data_type']]['parameters'][p]['id']) + '/'
                    if not os.path.exists(path):
                        os.makedirs(path)
                    fn = str(self.args['scenario_id']) + '_' + str(self.conf[args['data_type']]['parameters'][p]['id']) + '_' + str(y) + '.h5'
                    print(path + fn)
                    #sys.exit()
                    try:
                        df_final[['values_block_0']].to_hdf(path + fn, key='table' , mode='w', format='table')
                    except:
                        sys.exit("ERROR: " + str(sys.exc_info()))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May  1 18:23:45 2017

@author: David.Tew
"""

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm

from EnergyMarketTools.DGValueProp import SensitivityStudy
from EnergyMarketTools.EIADataQuery import EIAStateQuery
from EnergyMarketTools.parula_cmap import parula_cmap

class MarketStudy:
    """Run a two parameter sensitivity study in the specified regions, over
    the specified years, and for the specified sectors. A study portfolio is
    returned in the form of a triply nested dictionary."""
    def __init__(self,Regions=['US'],Years=[2015],Sectors=['COM'],
                     x_param_name=None,
                     x_param_values=None,
                     y_param_name=None,
                     y_param_values=None,
                     electric_efficiency = 0.7,
                     thermal_efficiency = '(1-electric_efficiency)/2',
                     installed_price_dol_W = 3,
                     maint_cost_dol_kWh = 0.02,
                     life_yrs = 20,
                     elec_capacity_utilization = 0.80,
                     therm_capacity_utilization = 0.25,
                     baseline_thermal_efficiency = 0.9,
                     inflation_rate = 0.02,
                     discount_rate = 0.15,
                     grid_baseline_elec_efficiency=0.34):

        self.Sectors = Sectors
        self.Years = Years
        self.Regions = Regions
        self.grid_baseline_elec_efficiency = grid_baseline_elec_efficiency

        # Array of zeros for initialization purposes
        SensZeros = np.zeros((len(y_param_values),len(x_param_values)))

        # Empty dictionary for market study results
        self.studyPortfolio = {sector: {year: {region: None for region in Regions}
            for year in Years} for sector in Sectors}

        self.TotalMarket_TWh = {sector: {year: 0 for year in Years} for sector in Sectors}
        self.AddressedMarket_TWh = {sector: {year: SensZeros for year in Years} for sector in Sectors}
        self.TotalMarketPen = {sector: {year: SensZeros for year in Years} for sector in Sectors}
        self.PrimaryEnergySavings_TWh = {sector: {year: SensZeros for year in Years} for sector in Sectors}
        self.PrimaryEnergySavings_Quads = {sector: {year: SensZeros for year in Years} for sector in Sectors}

        for sector in Sectors:
            for year in Years:
                for region in Regions:

                    self.studyPortfolio[sector][year][region] = \
                             SensitivityStudy(Region=region, Year=year, Sector=sector,
                             x_param_name=x_param_name,
                             x_param_values=x_param_values,
                             y_param_name=y_param_name,
                             y_param_values=y_param_values,
                             electric_efficiency = electric_efficiency,
                             thermal_efficiency = thermal_efficiency,
                             installed_price_dol_W = installed_price_dol_W,
                             maint_cost_dol_kWh = maint_cost_dol_kWh,
                             life_yrs = life_yrs,
                             elec_capacity_utilization = elec_capacity_utilization,
                             therm_capacity_utilization = therm_capacity_utilization,
                             baseline_thermal_efficiency = baseline_thermal_efficiency,
                             inflation_rate = inflation_rate,
                             discount_rate = discount_rate)

                    region_TotalMarket_TWh = self.studyPortfolio[sector][year][region].NetGeneration_TWh
                    region_AddressedMarket_TWh = region_TotalMarket_TWh * \
                           self.studyPortfolio[sector][year][region].get_val_metric('MaxMarketPen')

                    if self.TotalMarket_TWh[sector][year] == 0:
                        self.TotalMarket_TWh[sector][year] = region_TotalMarket_TWh
                        self.AddressedMarket_TWh[sector][year] = region_AddressedMarket_TWh
                    else:
                        self.TotalMarket_TWh[sector][year] += region_TotalMarket_TWh
                        self.AddressedMarket_TWh[sector][year] += region_AddressedMarket_TWh

                    #print('Total Market (TWh) = %s' % self.TotalMarket_TWh)

                self.TotalMarketPen[sector][year] = self.AddressedMarket_TWh[sector][year] / \
                                    self.TotalMarket_TWh[sector][year]

                self.PrimaryEnergySavings_TWh[sector][year] = \
                                     self.AddressedMarket_TWh[sector][year]/self.grid_baseline_elec_efficiency * ( 1 -
                                     self.grid_baseline_elec_efficiency /
                                     self.studyPortfolio[sector][year][region].get_val_metric('Effective_Electric_Efficiency') )

                self.PrimaryEnergySavings_Quads[sector][year] = self.PrimaryEnergySavings_TWh[sector][year] / 293.1

    def contour_plot(self,sector,year,region=None,metric=None):
        """Market penetration contour plot."""

        if metric in dir(self.studyPortfolio[sector][year][region].study[0][0].out):

            """Draw a contour plot for the specified metric in the specified
            sector, year and region."""
            fig, ax = self.studyPortfolio[sector][year][region].contour_plot(metric)

        else:
            """Draw total market penetration contours."""
            z_param = getattr(self,metric)[sector][year]

            figtitle = metric + ' ' + sector + ' ' + str(year)
            fig = plt.figure(num=figtitle)

            ax = plt.subplot(1,1,1)

            cmap = cm.jet #cm.PRGn

            if metric is 'TotalMarketPen':
                levels = np.linspace(0,1,21)
            else:
                levels = np.linspace(np.amin(z_param),np.amax(z_param),21)

            norm = cm.colors.Normalize(vmax=levels.max(), vmin=levels.min())

            plt.contourf(self.studyPortfolio[sector][year][region].x_param_values,
                         self.studyPortfolio[sector][year][region].y_param_values,
                         z_param,levels, cmap=cm.get_cmap(cmap, len(levels)-1),norm=norm)

            plt.colorbar(format='%.1f'); plt.autoscale(False)

            plt.xlabel(self.studyPortfolio[sector][year][region].pretty_label(
                    self.studyPortfolio[sector][year][region].x_param_name))
            plt.ylabel(self.studyPortfolio[sector][year][region].pretty_label(
                    self.studyPortfolio[sector][year][region].y_param_name))
            #plt.title(figtitle)

            plt.contour(self.studyPortfolio[sector][year][region].x_param_values,
                        self.studyPortfolio[sector][year][region].y_param_values,
                        z_param,levels,linewidths=0.25,colors='k')

            # Draw 1 Quad/year contour
            z_Quad = getattr(self,'PrimaryEnergySavings_Quads')[sector][year]
            plt.contour(self.studyPortfolio[sector][year][region].x_param_values,
                        self.studyPortfolio[sector][year][region].y_param_values,
                        z_Quad,levels=[1],linewidths=1,linestyles='dashed',colors='w')

            # Draw 50% market penetration contour
            z_Quad = getattr(self,'TotalMarketPen')[sector][year]
            plt.contour(self.studyPortfolio[sector][year][region].x_param_values,
                        self.studyPortfolio[sector][year][region].y_param_values,
                        z_Quad,levels=[0.5],linewidths=1,linestyles='dotted',colors='r')

            plt.tight_layout()

        return fig, ax


import fixtures
from oslo_log import log as logging
from oslotest import base as test_base
import testtools

from ec2api import exception
from ec2api.tests.unit import base
from ec2api.tests.unit import tools

class TestToolsTestCase(testtools.TestCase):

    def test_update_dict(self):
        d1 = {'a': 1, 'b': 2}
        d2 = {'b': 22, 'c': 33}
        res = tools.update_dict(d1, {})
        self.assertEqual({'a': 1, 'b': 2}, res)
        res = tools.update_dict(d1, d2)
        self.assertEqual({'a': 1, 'b': 22, 'c': 33}, res)
        self.assertEqual({'a': 1, 'b': 2}, d1)

    def test_purge_dict(self):
        d1 = {'a': 1, 'b': 2, 'c': 3}
        res = tools.purge_dict(d1, ())
        self.assertEqual({'a': 1, 'b': 2, 'c': 3}, res)
        res = tools.purge_dict(d1, ('b', 'c'))
        self.assertEqual({'a': 1}, res)
        self.assertEqual({'a': 1, 'b': 2, 'c': 3}, d1)

    def test_patch_dict(self):
        d1 = {'a': 1, 'b': 2, 'c': 3}
        d2 = {'c': 33, 'd': 44, 'e': 55}
        res = tools.patch_dict(d1, d2, ('b', 'e'))
        self.assertEqual({'a': 1, 'c': 33, 'd': 44}, res)
        self.assertEqual({'a': 1, 'b': 2, 'c': 3}, d1)

class TestBaseTestCase(base.ApiTestCase):

    def test_validate_exception_format_is_enabled_for_tests(self):
        with tools.ScreeningLogger():
            self.assertRaises(KeyError, exception.InvalidVpcRange,
                              fake='value')
        with tools.ScreeningLogger():
            self.assertRaises(TypeError, exception.InvalidID, {'id': 'value'})

class LoggingTestCase(test_base.BaseTestCase):

    def test_hide_logs(self):
        with fixtures.FakeLogger() as logger:
            with tools.ScreeningLogger():
                LOG = logging.getLogger('ec2api.api')
                LOG.critical('critical message')
                LOG.error('error message')
                LOG.warning('warning message')
            self.assertEqual(0, len(logger.output))

    def test_screen_logs(self):
        with fixtures.FakeLogger() as logger:
            with tools.ScreeningLogger(log_name='ec2api.api'):
                LOG1 = logging.getLogger('ec2api.api')
                LOG1.error('error message')
                LOG2 = logging.getLogger('ec2api.api.vpc')
                LOG2.warning('warning message')
            self.assertIn('warning message', logger.output)
            self.assertNotIn('error message', logger.output)

    def test_show_logs_on_unhandled_exception(self):
        with fixtures.FakeLogger() as logger:
            try:
                with tools.ScreeningLogger():
                    LOG = logging.getLogger('ec2api.api')
                    LOG.error('error message')
                    raise Exception()
            except Exception:
                pass
            self.assertIn('error message', logger.output)


import torch
import torchvision.transforms as transforms
import torch.nn as nn
import torchvision
from PIL import Image
from glob import glob
from Print_Correct_Label import Print_Correct_Label
import random
import warnings

# bynum = [3391, 4982, 3153, 2088, 3530, 3898, 2447, 3725, 1380, 3815,
#          1342, 2960, 4238, 1700, 4659, 2206, 2511, 2664, 4939, 1974]

radom_times = 20
image_num = -1  # 指定对象输入, 随机时此处填-1
Class = 'PROP'
Version = 1
Epoch = 9
break_on = 0  # 遇错终止功能

break_flag = 0

# 忽略特定警告类型
warnings.filterwarnings("ignore", category=UserWarning)

# 获取所有样本号
all_src_file_list = glob('D:/python_pro/pgmdata/*')
src_labels = []
for i in range(len(all_src_file_list)):
    src_labels.append(all_src_file_list[i].split('\\')[-1])
    src_labels[i] = src_labels[i].split('.')[0]

# 获取类标签
src_file_list = glob('D:/python_pro/' + Class + '/ALL/*')
class_labels = []
for i in range(len(src_file_list)):
    class_labels.append(src_file_list[i].split('\\')[-1])

for kk in range(radom_times):
    if kk == 0:
        if image_num == -1:
            image_num = random.choice(src_labels)
            while ' ' not in Print_Correct_Label(image_num, Class, 0):
                image_num = random.choice(src_labels)
        else:
            break_flag = 1
    else:
        image_num = random.choice(src_labels)
        while ' ' not in Print_Correct_Label(image_num, Class, 0):
            image_num = random.choice(src_labels)

    # image_num = bynum[kk]

    # 加载预训练的ResNet模型
    model = torchvision.models.resnet18(pretrained=True)
    num_classes = len(class_labels)
    model.fc = nn.Linear(model.fc.in_features, num_classes)  # 替换最后一层全连接层
    model.load_state_dict(torch.load(f'D:/大学/课程/模式识别/课程项目/人脸图像识别/python/MODEL/' + Class + '/' + Class +
                                     f'_ResNet18_V{Version}/ResNet18_V{Version}_Epoch{Epoch}.pth'))
    model.eval()

    # 图像预处理
    transform = transforms.Compose([
        transforms.Resize((128, 128)),
        transforms.Grayscale(num_output_channels=3),  # 将单通道图像转换为3通道
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # 使用与训练时相同的均值和标准差
    ])

    # 加载待预测的图像
    image_path = f'D:/python_pro/pgmdata/{image_num}.pgm'
    image = Image.open(image_path)
    input_image = transform(image).unsqueeze(0)  # 在维度0上添加一个批次维度

    # 预测图像类别
    if Class == 'SEX':
        with torch.no_grad():
            output = model(input_image)
            probabilities = torch.sigmoid(output)
            _, predicted_labels = probabilities.topk(2)  # 获取预测概率最高的2个类别
    else:
        with torch.no_grad():
            output = model(input_image)
            probabilities = torch.sigmoid(output)
            _, predicted_labels = probabilities.topk(3)  # 获取预测概率最高的3个类别

    correct_label = Print_Correct_Label(image_num, Class, 0)

    # 打印预测结果
    print(f'\033[1m\033[33m\033[3mLabel_{kk + 1}:\t{image_num}\033[0m', end='\t\t\t\t\t\t\t')
    if probabilities[0, predicted_labels[0, 1]].item() < 0.91:
        # print(probabilities[0, predicted_labels[0, 1]].item())
        if class_labels[predicted_labels[0, 0]] == correct_label:
            print('\033[1m\033[32mCorrect\033[0m')
            for i in range(predicted_labels.size(1)):
                class_index = predicted_labels[0, i]
                class_label = class_labels[class_index]
                probability = probabilities[0, class_index]
                print(f"Predicted label: {class_label}, Probability: {probability.item():.4f}")
        else:
            print('\033[1m\033[31mWrong\033[0m')
            for i in range(predicted_labels.size(1)):
                class_index = predicted_labels[0, i]
                class_label = class_labels[class_index]
                probability = probabilities[0, class_index]
                print(f"\033[41mPredicted label: {class_label}, Probability: {probability.item():.4f}\033[0m")
            if break_on:
                break_flag = 1
    else:
        if (class_labels[predicted_labels[0, 0]] in correct_label) and (class_labels[predicted_labels[0, 1]] in correct_label):
            print('\033[1m\033[32mCorrect\033[0m')
            for i in range(predicted_labels.size(1)):
                class_index = predicted_labels[0, i]
                class_label = class_labels[class_index]
                probability = probabilities[0, class_index]
                print(f"Predicted label: {class_label}, Probability: {probability.item():.4f}")
        else:
            print('\033[1m\033[31mWrong\033[0m')
            for i in range(predicted_labels.size(1)):
                class_index = predicted_labels[0, i]
                class_label = class_labels[class_index]
                probability = probabilities[0, class_index]
                print(f"\033[41mPredicted label: {class_label}, Probability: {probability.item():.4f}\033[0m")
            if break_on:
                break_flag = 1

    print(f'Correct label: {correct_label}')

    if break_flag:
        break


# %%
import pickle
import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial.distance import pdist
from scipy.cluster.hierarchy import dendrogram, linkage
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
import os.path as osp
import argparse
import scipy.stats
import pandas as pd
import seaborn as sns

plt.rcParams["font.sans-serif"] = ["SimHei"]  # 用来正常显示中文标签
plt.rcParams["axes.unicode_minus"] = False  # 用来正常显示负号

#%%
clippredpkl_file = '/mnt/liying.cibr.ac.cn_Data_Temp/multiview_9/chenxf/00_BehaviorAnalysis-seq2seq/SexAge/DayAll20230828/All-DecSeq/FWPCA0.00_P100_en3_hid30_epoch300-decSeqPC0.9_svm2allAcc0.96_kmeansK2use-43_fromK1-20_K100.clippredpkl'
beh_project_dir = osp.dirname(clippredpkl_file)
clippreddata = pickle.load(open(clippredpkl_file, "rb"))
assert {
    "ncluster",
    "ntwin",
    "cluster_labels",
    "embedding",
    "embedding_d2",
    "clipNames",
    "cluster_names",
} <= clippreddata.keys()
# %% 载入数据
feat_data = clippreddata["embedding"]  # nsample x nfeat

# start from 1. The 0 is nonsocial
kmeans_label = clippreddata["cluster_labels"]
n_kmeans = clippreddata["ncluster"]
cluster_names = clippreddata['cluster_names']
assert kmeans_label.min() == 1

def get_pearsonr(xs, ys, embedding):
    return np.array([scipy.stats.pearsonr(embedding[x], embedding[y])[0] for x, y in zip(xs, ys)])

n_each_class = 200
n_blockin = 4
clusterlabel = clippreddata["cluster_labels"] # kmeans_data['labels'] + 1
assert clusterlabel.min()==1 and clusterlabel.max()==n_kmeans
uni_id, uni_c = np.unique(clusterlabel, return_counts=True)
np.random.seed(1000)

n_each_class = np.min([uni_c.min()//2, 200])

id_ego_choose_each_class = np.zeros((len(uni_id)+1, n_each_class, n_blockin), dtype=int)
id_partner_choose_each_class = np.zeros_like(id_ego_choose_each_class)
id_nchoose_each_class = np.zeros_like(id_ego_choose_each_class)
for i in uni_id:
    id_list = np.where(clusterlabel==i)[0]

    id_choose = np.stack([np.random.choice(id_list, n_blockin*2, replace=True) for _ in range(n_each_class)])
    id_ego_choose_each_class[i] = id_choose[...,  :n_blockin]
    id_partner_choose_each_class[i] = id_choose[...,  n_blockin:]

    id_list = np.where(clusterlabel!=i)[0]
    id_nchoose = np.stack([np.random.choice(id_list, n_blockin, replace=True) for _ in range(n_each_class)])
    id_nchoose_each_class[i] = id_nchoose

df = pd.DataFrame(columns=['r', 'group', 'cluster'])
df_list = []
for i in uni_id:
    xego = id_ego_choose_each_class[i]
    xpartner = id_partner_choose_each_class[i]
    yother = id_nchoose_each_class[i]
    rs_intra = [get_pearsonr(xego_, xpartner_, feat_data).mean() for (xego_, xpartner_) in zip(xego, xpartner)]
    rs_inter = [get_pearsonr(xego_, yother_, feat_data).mean() for (xego_, yother_) in zip(xego, yother)]
    df1 = pd.DataFrame(zip(rs_intra, ['intra']*n_each_class, np.zeros((n_each_class,), dtype=int)+i))
    df2 = pd.DataFrame(zip(rs_inter, ['inter']*n_each_class, np.zeros((n_each_class,), dtype=int)+i))
    df_list.extend([df1, df2])

df = pd.concat(df_list)
df.columns = ['r', 'group', 'cluster']
df_mean = df.groupby(by=['group', 'cluster']).agg('median').reset_index()
df_mean.loc[(df_mean['group'] =='intra'), 'cluster'] -= 1.2
df_mean.loc[(df_mean['group'] =='inter'), 'cluster'] -= 0.8

plt.figure(figsize= (20,8))
palette = ['#a4cee4', '#e6e6e6']
sns.violinplot(data=df, x='cluster', y='r', scale='count',
               palette=palette, cut = 5,
               inner=None, hue='group')

plt.plot(df_mean['cluster'].values, df_mean['r'].values, 'wo', markeredgecolor='k')
plt.xlim(-0.5, n_kmeans - 0.5)
plt.ylim(-1, 1)
plt.yticks([-1, -0.5, 0, 0.5, 1])
plt.ylabel('Correlation coefficient')
outfig = osp.join(beh_project_dir, 'violinplot_cluster_similarity_plot.pdf')
plt.savefig(outfig)
#%%
K = len(uni_id)
matrix = np.zeros((K,K))
for i in range(K):
    for j in range(K):
        xego = id_ego_choose_each_class[i+1][...,0]
        xpartner = id_partner_choose_each_class[j+1][...,0]
        matrix[i, j] = get_pearsonr(xego, xpartner, feat_data).mean()

matrix_mirror = (matrix + matrix.T)/2

leaf_labels = [
    f"[{leaf+1:>2}] {label.rstrip()}" for leaf, label in enumerate(cluster_names)
]

df_matrix_mirror = pd.DataFrame(matrix_mirror, columns=np.arange(1,K+1), index=np.arange(1,K+1))
df_matrix_mirror = pd.DataFrame(matrix_mirror, columns=np.arange(1,K+1), index=leaf_labels)
sns.clustermap(df_matrix_mirror, cmap="vlag", vmin=-1, vmax=1,
               xticklabels=1,yticklabels=1,
               method='complete',
               figsize=(11,8) )
outfig = osp.join(osp.dirname(clippredpkl_file), "dendrogram_heatmap_similiar.pdf")
plt.savefig(outfig, bbox_inches="tight")

dist = pdist(matrix_mirror)
linkage_matrix = linkage(dist,
                         method='complete',
                         )

fig, ax = plt.subplots(figsize=(4, 14))
plt.rcParams["font.family"] = "Arial"
dendrogram_result = dendrogram(
    linkage_matrix,
    ax=ax,
    labels=leaf_labels,
    orientation="left",
    color_threshold=0,
    above_threshold_color="k",
)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["left"].set_visible(False)
ax.set_ylim(ax.get_ylim()[::-1])
plt.xticks([])
plt.yticks(fontsize=14)

outfig = osp.join(osp.dirname(clippredpkl_file), "dendrogram_similiar.pdf")
plt.savefig(outfig, bbox_inches="tight")


import numpy as np
import os, sys, ipdb, datetime, click, pickle #tqdm

import seaborn as sns
sns.set_context('talk')
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm

"""
Written by Christopher Hsu

Plot and save figure 4 right
Na != Nd
"""

src_dir = './fig4/data'
w = pickle.load(open(os.path.join(src_dir, 'w_diff.pkl'),'rb'))

Nas = w['Nas']
Nds = w['Nds']
costs = w['costs']
stdev = w['stdev']

plt.rcParams['figure.figsize'] = [9,8]
sns.set_theme()
fsz = 32
plt.rc('font', size=fsz)
plt.rc('axes', titlesize=fsz)
plt.rc('axes', labelsize=fsz)
plt.rc('xtick', labelsize=fsz)
plt.rc('ytick', labelsize=fsz)
plt.rc('legend', fontsize=fsz)
plt.rc('figure', titlesize=fsz)
plt.rc('pdf', fonttype=42)
sns.set_style("ticks", rc={"axes.grid":True})

fig1 = plt.figure()
ax = fig1.add_subplot()

cmap = sns.color_palette("rocket_r", as_cmap=True)
X = np.arange(1,100,5)
Y = np.arange(1,100,5)
cp = ax.contourf(X,Y,costs.reshape(20,20).T,levels=[5,10,20,40,80,160],cmap=cmap, norm=LogNorm())

ax.set_xlabel("$\sum_a N_a$ (# of Attackers)")
ax.set_ylabel("$\sum_d N_d$ (# of Defenders)")

cbar = fig1.colorbar(cp)
cbar.ax.set_ylabel('Mean Harm')

save_dir = './fig4/figures'
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

fig1.savefig(os.path.join(save_dir,'limits_diff.pdf'),bbox_inches='tight')

plt.show()


import scrapy

from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class VivarealSpider(CrawlSpider):

    name = 'vivareal_crawl'
    start_urls = ['https://www.vivareal.com.br/venda/pernambuco/recife/bairros/boa-viagem/apartamento_residencial/']
    rules = (
        Rule(
            LinkExtractor(allow='/venda/pernambuco/recife/bairros/boa-viagem/apartamento_residencial/')
        ),
        Rule(
            LinkExtractor(
                allow='/imovel/',
            ), callback='parse_imovel'
        )
    )

    def parse_imovel(self, response):
        yield {
            'title': response.xpath("//title/text()").extract_first()
        }

# yield {
#             'title': response.xpath("//title/text()").extract_first(),
#             'price': response.xpath("//*[contains(@class, 'js-detail-sale-price')]/text()").extract_first(),
#             'area': response.xpath("//*[contains(@class, 'js-detail-area-value')]/text()").extract_first(),
#             'latitude': response.xpath("//meta[contains(@property, 'place:location:latitude')]/@content").extract_first(),
#             'longitude': response.xpath("//meta[contains(@property, 'place:location:longitude')]/@content").extract_first(),
#         }


import os.path
import shutil
from datetime import datetime
from pathlib import Path
import time
import hydra
import wandb
import numpy as np
import torch
from omegaconf import DictConfig, OmegaConf
from torch.cuda import amp
import torch.multiprocessing as mp
import torch.distributed as dist
from tqdm import tqdm
from torch.optim.lr_scheduler import LambdaLR

from dataload.data_load import load_data
from dataload.data_preprocess import prepare_preprocessed_data
from utils.utils import evaluate
from utils.metrics import *
from utils.common import *
from transformers import BertTokenizer
from transformers import AdamW
from torch.nn.parallel import DistributedDataParallel as DDP

### custom your wandb setting here ###
# os.environ["WANDB_API_KEY"] = ""
os.environ["WANDB_MODE"] = "offline"

def setup(rank, world_size):
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '23342'
    dist.init_process_group("nccl", rank=rank, world_size=world_size)

def cleanup():
    dist.destroy_process_group()

def train(model, optimizer, dataloader, local_rank, world_size, cfg):
    model.train()
    torch.set_grad_enabled(True)

    mean_loss = torch.zeros(2).to(local_rank)
    acc_cnt = torch.zeros(2).to(local_rank)
    acc_cnt_pos = torch.zeros(2).to(local_rank)
    for cnt, (
            subgraph, mapping_idx, candidate_news, candidate_entity, entity_mask, batch_enc, batch_token, batch_attn,
            target,
            imp) \
            in enumerate(tqdm(dataloader,
                              total=int(cfg.num_epochs * (cfg.dataset.pos_count // cfg.batch_size + 1)),
                              desc=f"[{local_rank}] Training"), start=1):
        subgraph = subgraph.to(local_rank, non_blocking=True)
        mapping_idx = mapping_idx.to(local_rank, non_blocking=True)
        candidate_news = candidate_news.to(local_rank, non_blocking=True)
        imp = imp.to(local_rank, non_blocking=True)
        candidate_entity = candidate_entity.to(local_rank, non_blocking=True)
        entity_mask = entity_mask.to(local_rank, non_blocking=True)

        batch_enc = batch_enc.to(local_rank, non_blocking=True)
        batch_token = batch_token.to(local_rank, non_blocking=True)
        batch_attn = batch_attn.to(local_rank, non_blocking=True)
        target = target.to(local_rank, non_blocking=True)

        # batch_attn = batch_attn.to(local_rank, non_blocking=True)
        # batch_labs = batch_labs.to(local_rank, non_blocking=True)
        # batch_token = batch_token.to(local_rank, non_blocking=True)
        # '[P1][P2][P3][nsep] 0 [nsep] 1 [nsep] 2 [nsep] 3 [nsep] 4 [nsep] 5[SEP][Q1][Q2][Q3]0[SEP]Does the user click the news
        # sentence = [[d["sentence"] for d in sublist] for sublist in batch_enc]

        loss, scores = model(subgraph, mapping_idx, candidate_news, candidate_entity, entity_mask, batch_enc,
                             batch_token, batch_attn, target, imp)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        print("loss: ", loss.item())
        print("score: ", scores[10])
        mean_loss[0] += loss.item()
        mean_loss[1] += 1

        predict = torch.argmax(scores.detach(), dim=1)
        num_correct = (predict == target).sum()
        acc_cnt[0] += num_correct
        acc_cnt[1] += predict.size(0)

        positive_idx = torch.where(target == 1)[0]
        num_correct_pos = (predict[positive_idx] == target[positive_idx]).sum()
        acc_cnt_pos[0] += num_correct_pos
        acc_cnt_pos[1] += positive_idx.size(0)

    dist.all_reduce(mean_loss, op=dist.ReduceOp.SUM)
    dist.all_reduce(acc_cnt, op=dist.ReduceOp.SUM)
    dist.all_reduce(acc_cnt_pos, op=dist.ReduceOp.SUM)

    loss_epoch = mean_loss[0] / mean_loss[1]
    acc = acc_cnt[0] / acc_cnt[1]
    acc_pos = acc_cnt_pos[0] / acc_cnt_pos[1]
    pos_ratio = acc_cnt_pos[1] / acc_cnt[1]

    return loss_epoch.item(), acc.item(), acc_pos.item(), pos_ratio.item()

def eval(model, local_rank, dataloader, world_size, cfg):
    model.eval()
    # data_loader = tqdm(data_loader)
    val_scores = []
    acc_cnt = torch.zeros(2).to(local_rank)
    acc_cnt_pos = torch.zeros(2).to(local_rank)
    imp_ids = []
    labels = []
    for cnt, (
            subgraph, mapping_idx, clicked_entity, candidate_news, candidate_entity, entity_mask, batch_enc,
            batch_token, batch_attn, target,
            imp) \
            in enumerate(tqdm(dataloader,
                              total=int(cfg.num_epochs * (cfg.dataset.pos_count // cfg.batch_size + 1)),
                              desc=f"[{local_rank}] Training"), start=1):
        imp_ids = imp_ids + imp
        labels = labels + target.cpu().numpy().tolist()

        subgraph = subgraph.to(local_rank, non_blocking=True)
        mapping_idx = mapping_idx.to(local_rank, non_blocking=True)
        candidate_news = candidate_news.to(local_rank, non_blocking=True)
        imp = imp.to(local_rank, non_blocking=True)
        candidate_entity = candidate_entity.to(local_rank, non_blocking=True)
        entity_mask = entity_mask.to(local_rank, non_blocking=True)

        batch_enc = batch_enc.to(local_rank, non_blocking=True)
        batch_token = batch_token.to(local_rank, non_blocking=True)
        batch_attn = batch_attn.to(local_rank, non_blocking=True)
        target = target.to(local_rank, non_blocking=True)

        # batch_attn = batch_attn.to(local_rank, non_blocking=True)
        # batch_labs = batch_labs.to(local_rank, non_blocking=True)
        # batch_token = batch_token.to(local_rank, non_blocking=True)
        # '[P1][P2][P3][nsep] 0 [nsep] 1 [nsep] 2 [nsep] 3 [nsep] 4 [nsep] 5[SEP][Q1][Q2][Q3]0[SEP]Does the user click the news
        # sentence = [[d["sentence"] for d in sublist] for sublist in batch_enc]

        loss, scores = model.module.validation_process(subgraph, mapping_idx, clicked_entity, candidate_news,
                                                       candidate_entity, entity_mask, batch_enc,
                                                       batch_token, batch_attn, target, imp)
        ranking_scores = scores[:, 1].detach()
        val_scores.append(ranking_scores)

        predict = torch.argmax(scores.detach(), dim=1)
        num_correct = (predict == target).sum()
        acc_cnt[0] += num_correct
        acc_cnt[1] += predict.size(0)

        positive_idx = torch.where(target == 1)[0]
        num_correct_pos = (predict[positive_idx] == target[positive_idx]).sum()
        acc_cnt_pos[0] += num_correct_pos
        acc_cnt_pos[1] += positive_idx.size(0)

    dist.all_reduce(acc_cnt, op=dist.ReduceOp.SUM)
    dist.all_reduce(acc_cnt_pos, op=dist.ReduceOp.SUM)

    acc = acc_cnt[0] / acc_cnt[1]
    acc_pos = acc_cnt_pos[0] / acc_cnt_pos[1]
    pos_ratio = acc_cnt_pos[1] / acc_cnt[1]

    val_scores = torch.cat(val_scores, dim=0)
    val_impids = torch.IntTensor(imp_ids).to(local_rank)
    val_labels = torch.IntTensor(labels).to(local_rank)

    val_scores_list = [torch.zeros_like(val_scores).to(local_rank) for _ in range(world_size)]
    val_impids_list = [torch.zeros_like(val_impids).to(local_rank) for _ in range(world_size)]
    val_labels_list = [torch.zeros_like(val_labels).to(local_rank) for _ in range(world_size)]

    dist.all_gather(val_scores_list, val_scores)
    dist.all_gather(val_impids_list, val_impids)
    dist.all_gather(val_labels_list, val_labels)

    return val_scores_list, acc.item(), acc_pos.item(), pos_ratio.item(), val_impids_list, val_labels_list

def load_tokenizer(cfg):
    tokenizer = BertTokenizer.from_pretrained(cfg.token.bertmodel)
    conti_tokens1 = []
    for i in range(cfg.token.num_conti1):
        conti_tokens1.append('[P' + str(i + 1) + ']')
    conti_tokens2 = []
    for i in range(cfg.token.num_conti2):
        conti_tokens2.append('[Q' + str(i + 1) + ']')

    new_tokens = ['[NSEP]']
    tokenizer.add_tokens(new_tokens)

    conti_tokens = conti_tokens1 + conti_tokens2
    tokenizer.add_tokens(conti_tokens)

    new_vocab_size = len(tokenizer)
    cfg.token.vocab_size = new_vocab_size
    # cfg.token.tokenizer = tokenizer
    return tokenizer, conti_tokens1, conti_tokens2

def main_worker(local_rank, cfg):
    # -----------------------------------------Environment Initial
    seed_everything(cfg.seed)  # 设置随机数种子
    dist.init_process_group(backend='nccl',
                            init_method='tcp://127.0.0.1:23456',
                            world_size=cfg.gpu_num,
                            rank=local_rank)  # 表明其在整个分布式环境中的排名和标识
    world_size = cfg.gpu_num

    # -----------------------------------------Dataset & Model Load
    num_training_steps = int(cfg.num_epochs * cfg.dataset.pos_count / (cfg.batch_size * cfg.accumulation_steps))
    num_warmup_steps = int(num_training_steps * cfg.warmup_ratio + 1)

    tokenizer, conti_tokens1, conti_tokens2 = load_tokenizer(cfg)
    conti_tokens = [conti_tokens1, conti_tokens2]

    # val_dataloader = load_data(cfg, mode='val',local_rank=local_rank, tokenizer=tokenizer, conti_tokens=conti_tokens)
    model = load_model(cfg, tokenizer).to(local_rank)
    net = DDP(model, device_ids=[local_rank])
    # optimizer = torch.optim.Adam(model.parameters(), lr=cfg.optimizer.lr)

    # AdamW
    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in net.named_parameters() if not any(nd in n for nd in no_decay)],
         'weight_decay': cfg.train.wd},
        {'params': [p for n, p in net.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
    ]
    optimizer = AdamW(params=optimizer_grouped_parameters, lr=cfg.train.lr)

    # lr_lambda = lambda step: 1.0 if step > num_warmup_steps else step / num_warmup_steps
    # scheduler = LambdaLR(optimizer, lr_lambda)

    # ------------------------------------------Load Checkpoint & optimizer

    # model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[local_rank])

    train_dataloader = load_data(cfg, mode='train', model=net, local_rank=local_rank, tokenizer=tokenizer,
                                 conti_tokens=conti_tokens)  # 加载训练数据
    # val_dataloader = load_data(cfg, mode='val', model=net, local_rank=local_rank, tokenizer=tokenizer,
    #                              conti_tokens=conti_tokens)
    metrics = ['auc', 'mrr', 'ndcg5', 'ndcg10']
    best_val_result = {}
    best_val_epoch = {}
    for m in metrics:
        best_val_result[m] = 0.0
        best_val_epoch[m] = 0

    for epoch in range(cfg.num_epochs):
        # #################################  train  ###################################
        st_tra = time.time()
        if local_rank == 0:
            print('--------------------------------------------------------------------')
            print('start training: ', datetime.now())
            print('Epoch: ', epoch)
            print('lr:', optimizer.state_dict()['param_groups'][0]['lr'])
        loss, acc_tra, acc_pos_tra, pos_ratio_tra = \
            train(net, optimizer, train_dataloader, local_rank, world_size, cfg)
        # scheduler.step()

        end_tra = time.time()
        train_spend = (end_tra - st_tra) / 3600
        if local_rank == 0:
            print("Train Loss: %0.4f" % loss)
            print("Train ACC: %0.4f\tACC-Positive: %0.4f\tPositiveRatio: %0.4f\t[%0.2f]" %
                  (acc_tra, acc_pos_tra, pos_ratio_tra, train_spend))
            if cfg.train.model_save:
                file = cfg.path.ckp_dir + '/Epoch-' + str(epoch) + '.pt'
                print('save file', file)
                torch.save(net.module.state_dict(), file)

        # #################################  val  ###################################
        st_val = time.time()
        val_scores, acc_val, acc_pos_val, pos_ratio_val, val_impids, val_labels = \
            eval(net, local_rank, val_dataloader, world_size, cfg)
        impressions = {}  # {1: {'score': [], 'lab': []}}
        for i in range(world_size):
            scores, imp_id, labs = val_scores[i], val_impids[i], val_labels[i]
            assert scores.size() == imp_id.size() == labs.size()
            scores = scores.cpu().numpy().tolist()
            imp_id = imp_id.cpu().numpy().tolist()
            labs = labs.cpu().numpy().tolist()
            for j in range(len(scores)):
                sco, imp, lab = scores[j], imp_id[j], labs[j]
                if imp not in impressions:
                    impressions[imp] = {'score': [], 'lab': []}
                    impressions[imp]['score'].append(sco)
                    impressions[imp]['lab'].append(lab)
                else:
                    impressions[imp]['score'].append(sco)
                    impressions[imp]['lab'].append(lab)

        predicts, truths = [], []
        for imp in impressions:
            sims, labs = impressions[imp]['score'], impressions[imp]['lab']
            sl_zip = sorted(zip(sims, labs), key=lambda x: x[0], reverse=True)
            sort_sims, sort_labs = zip(*sl_zip)
            predicts.append(list(range(1, len(sort_labs) + 1, 1)))
            truths.append(sort_labs)

        auc_val, mrr_val, ndcg5_val, ndcg10_val = evaluate(predicts, truths)
        if auc_val > best_val_result['auc']:
            best_val_result['auc'] = auc_val
            best_val_epoch['auc'] = epoch
        if mrr_val > best_val_result['mrr']:
            best_val_result['mrr'] = mrr_val
            best_val_epoch['mrr'] = epoch
        if ndcg5_val > best_val_result['ndcg5']:
            best_val_result['ndcg5'] = ndcg5_val
            best_val_epoch['ndcg5'] = epoch
        if ndcg10_val > best_val_result['ndcg10']:
            best_val_result['ndcg10'] = ndcg10_val
            best_val_epoch['ndcg10'] = epoch
        end_val = time.time()
        val_spend = (end_val - st_val) / 3600

        if local_rank == 0:
            print("Validate: AUC: %0.4f\tMRR: %0.4f\tnDCG@5: %0.4f\tnDCG@10: %0.4f\t[Val-Time: %0.2f]" %
                  (auc_val, mrr_val, ndcg5_val, ndcg10_val, val_spend))
            print('Best Result: AUC: %0.4f \tMRR: %0.4f \tNDCG@5: %0.4f \t NDCG@10: %0.4f' %
                  (best_val_result['auc'], best_val_result['mrr'], best_val_result['ndcg5'], best_val_result['ndcg10']))
            print('Best Epoch: AUC: %d \tMRR: %d \tNDCG@5: %d \t NDCG@10: %d' %
                  (best_val_epoch['auc'], best_val_epoch['mrr'], best_val_epoch['ndcg5'], best_val_epoch['ndcg10']))
        dist.barrier()
    if local_rank == 0:
        best_epochs = [best_val_epoch['auc'], best_val_epoch['mrr'], best_val_epoch['ndcg5'], best_val_epoch['ndcg10']]
        best_epoch = max(set(best_epochs), key=best_epochs.count)
        if cfg.train.model_save:
            old_file = cfg.path.ckp_dir + '/Epoch-' + str(best_epoch) + '.pt'
            # old_file = args.save_dir + '/Epoch-2.pt'
            if not os.path.exists('./temp'):
                os.makedirs('./temp')
            copy_file = './temp' + '/BestModel.pt'
            shutil.copy(old_file, copy_file)
            print('Copy ' + old_file + ' >>> ' + copy_file)

    if local_rank == 0:
        wandb.init(config=OmegaConf.to_container(cfg, resolve=True),
                   project=cfg.logger.exp_name, name=cfg.logger.run_name)
        print(model)

    if local_rank == 0:
        wandb.finish()
    cleanup()

@hydra.main(version_base="1.2", config_path=os.path.join(get_root(), "configs"), config_name="small")
def main(cfg: DictConfig):
    seed_everything(cfg.seed)
    # cfg.model='GLORY'
    # cfg.dataset='MINDSmall.yaml'
    # cfg.reprocess=True
    cfg.gpu_num = torch.cuda.device_count()
    prepare_preprocessed_data(cfg)
    mp.spawn(main_worker, nprocs=cfg.gpu_num, args=(cfg,))  # 这行代码的目的是使用 PyTorch 的分布式训练功能，以并行方式执行名为 main_worker 的函数。
    # 每个工作进程都会执行相同的 main_worker 函数，但可以根据配置中的参数来控制不同的行为

if __name__ == "__main__":
    main()


import os
import platform
import openai
import sys
import subprocess
import dotenv
import distro
import yaml
import pyperclip

from termcolor import colored
from colorama import init

def read_config() -> any:

  ## Find the executing directory (e.g. in case an alias is set)
  ## So we can find the config file
  yolo_path = os.path.abspath(__file__)
  prompt_path = os.path.dirname(yolo_path)

  config_file = os.path.join(prompt_path, "yolo.yaml")
  with open(config_file, 'r') as file:
    return yaml.safe_load(file)

# Construct the prompt
def get_full_prompt(user_prompt, shell):

  ## Find the executing directory (e.g. in case an alias is set)
  ## So we can find the prompt.txt file
  yolo_path = os.path.abspath(__file__)
  prompt_path = os.path.dirname(yolo_path)

  ## Load the prompt and prep it
  prompt_file = os.path.join(prompt_path, "prompt.txt")
  pre_prompt = open(prompt_file,"r").read()
  pre_prompt = pre_prompt.replace("{shell}", shell)
  pre_prompt = pre_prompt.replace("{os}", get_os_friendly_name())
  prompt = pre_prompt + user_prompt

  # be nice and make it a question
  if prompt[-1:] != "?" and prompt[-1:] != ".":
    prompt+="?"

  return prompt

def print_usage():
  print("Yolo v0.2.1 - by @wunderwuzzi23")
  print()
  print("Usage: yolo [-a] list the current directory information")
  print("Argument: -a: Prompt the user before running the command (only useful when safety is off)")
  print()

  print("Current configuration per yolo.yaml:")
  print("* Model        : " + str(config["model"]))
  print("* Temperature  : " + str(config["temperature"]))
  print("* Max. Tokens  : " + str(config["max_tokens"]))
  print("* Safety       : " + str(bool(config["safety"])))

def get_os_friendly_name():

  # Get OS Name
  os_name = platform.system()

  if os_name == "Linux":
    return "Linux/"+distro.name(pretty=True)
  elif os_name == "Windows":
    return os_name
  elif os_name == "Darwin":
    return "Darwin/macOS"
  else:
    return os_name

def set_api_key():
  # Two options for the user to specify they openai api key.
  #1. Place a ".env" file in same directory as this with the line:
  #   OPENAI_API_KEY="<yourkey>"
  #   or do `export OPENAI_API_KEY=<yourkey>` before use
  dotenv.load_dotenv()
  openai.api_key = os.getenv("OPENAI_API_KEY")

  #2. Place a ".openai.apikey" in the home directory that holds the line:
  #   <yourkey>
  #   Note: This options will likely be removed in the future
  if not openai.api_key:  #If statement to avoid "invalid filepath" error
    home_path = os.path.expanduser("~")
    openai.api_key_path = os.path.join(home_path,".openai.apikey")

  #3. Final option is the key might be in the yolo.yaml config file
  #   openai_apikey: <yourkey>
  if not openai.api_key:
    openai.api_key = config["openai_api_key"]

if __name__ == "__main__":

  config = read_config()
  set_api_key()

  # Unix based SHELL (/bin/bash, /bin/zsh), otherwise assuming it's Windows
  shell = os.environ.get("SHELL", "powershell.exe")

  command_start_idx  = 1     # Question starts at which argv index?
  ask_flag = False           # safety switch -a command line argument
  yolo = ""                  # user's answer to safety switch (-a) question y/n

  # Parse arguments and make sure we have at least a single word
  if len(sys.argv) < 2:
    print_usage()
    sys.exit(-1)

  # Safety switch via argument -a (local override of global setting)
  # Force Y/n questions before running the command
  if sys.argv[1] == "-a":
    ask_flag = True
    command_start_idx = 2

  # To allow easy/natural use we don't require the input to be a
  # single string. So, the user can just type yolo what is my name?
  # without having to put the question between ''
  arguments = sys.argv[command_start_idx:]
  user_prompt = " ".join(arguments)

def call_open_ai(query):
  # do we have a prompt from the user?
  if query == "":
      print ("No user prompt specified.")
      sys.exit(-1)

  # Load the correct prompt based on Shell and OS and append the user's prompt
  prompt = get_full_prompt(query, shell)

  # Make the first line also the system prompt
  system_prompt = prompt[1]
  #print(prompt)

  # Call the ChatGPT API
  response = openai.ChatCompletion.create(
    model=config["model"],
    messages=[
        {"role": "system", "content": system_prompt},
        {"role": "user", "content": prompt}
    ],
    temperature=config["temperature"],
    max_tokens=config["max_tokens"],
  )

  return response.choices[0].message.content.strip()

#Enable color output on Windows using colorama
init()

def check_for_issue(response):
  prefixes = ("sorry", "i'm sorry", "the question is not clear", "i'm", "i am")
  if response.lower().startswith(prefixes):
    print(colored("There was an issue: "+response, 'red'))
    sys.exit(-1)

def check_for_markdown(response):
  # odd corner case, sometimes ChatCompletion returns markdown
  if response.count("```",2):
    print(colored("The proposed command contains markdown, so I did not execute the response directly: \n", 'red')+response)
    sys.exit(-1)

def missing_posix_display():
  display = subprocess.check_output("echo $DISPLAY", shell=True)
  return display == b'\n'

def prompt_user_input(response):
  print("Command: " + colored(response, 'blue'))
  #print(config["safety"])

  if bool(config["safety"]) == True or ask_flag == True:
    prompt_text = "Execute command? [Y]es [n]o [m]odify [c]opy to clipboard ==> "
    if os.name == "posix" and missing_posix_display():
        prompt_text =  "Execute command? [Y]es [n]o [m]odify ==> "
    print(prompt_text, end = '')
    user_input = input()
    return user_input

  if config["safety"] == False:
     return "Y"

def evaluate_input(user_input, command):
  if user_input.upper() == "Y" or user_input == "":
    if shell == "powershell.exe":
      subprocess.run([shell, "/c", command], shell=False)
    else:
      # Unix: /bin/bash /bin/zsh: uses -c both Ubuntu and macOS should work, others might not
      subprocess.run([shell, "-c", command], shell=False)

  if user_input.upper() == "M":
    print("Modify prompt: ", end = '')
    modded_query = input()
    modded_response = call_open_ai(modded_query)
    check_for_issue(modded_response)
    check_for_markdown(modded_response)
    modded_user_input = prompt_user_input(modded_response)
    print()
    evaluate_input(modded_user_input, modded_response)

  if user_input.upper() == "C":
      if os.name == "posix" and missing_posix_display():
        return
      pyperclip.copy(command)
      print("Copied command to clipboard.")

res_command = call_open_ai(user_prompt)
check_for_issue(res_command)
check_for_markdown(res_command)
user_input = prompt_user_input(res_command)
print()
evaluate_input(user_input, res_command)


# Get Synonyms from WordNet
from nltk.corpus import wordnet

# synonyms = wordnet.synsets("world")

# print(synonyms[0].definition())
# print(synonyms[0].examples())

synonyms = []

for syn in wordnet.synsets('Computer'):

    for lemma in syn.lemmas():
        synonyms.append(lemma.name())

print(synonyms)


import PIL.Image
import cv2 as cv
import pandas as pd
import base64
from sklearn.neighbors import NearestNeighbors
import numpy as np
import streamlit as st
import PIL
import torch
from torchvision.models import resnet50

def preprocessing2(img):
  return torch.FloatTensor(cv.resize(img, (224, 224))).permute(2, 0, 1).unsqueeze(0) / 255.

def neiron_emb(img_path, model):
  img = cv.imread(img_path, cv.IMREAD_COLOR)
  img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
  inp = preprocessing2(img)
  with torch.no_grad():
      out = model(inp)
  emb = out[0].numpy()
  return emb

@st.cache(allow_output_mutation=True)
def loadBases():
    emb_db = './embedBaseDf11.csv'
    emb_base = pd.read_csv(emb_db, delimiter=',')
    emb_base['embed'] = emb_base['embed'].apply(
        lambda x: np.frombuffer(base64.b64decode(bytes(x[2:-1], encoding='ascii')), dtype=np.float32))
    return emb_base

def encodeImage(arr) -> np.ndarray:
    emds = []
    for i in arr:
        emb, _ = np.histogram(i, 2048)
        emds.append(emb)
    return emds

def workWithDb(img, classifier, db, db_neighbours):
    predhist = neiron_emb(img, classifier)
    distances, indices = db_neighbours.kneighbors(predhist.reshape(1, -1), return_distance=True)
    print(indices)
    return db.loc[indices[0], ['path']].values, distances[0]
    #return db.loc[indices[0]-2, ['path']].values, distances[0]

def main():
    loaded_model = resnet50(True).eval()
    database = loadBases()
    db_neighbours = NearestNeighbors(n_neighbors=5, metric='cosine')
    # print(type(database))
    emdbs = encodeImage(database['embed'].values)
    db_neighbours.fit(emdbs, y=list(range(len(database))))
    up_file = st.file_uploader('Choice file', type=['jpeg', 'jpg', 'webp', 'png', 'tiff'])
    if up_file is not None:
        try:
            img = PIL.Image.open(up_file)
            img = PIL.ImageOps.exif_transpose(img)
            img_paths, dists = workWithDb(np.array(img), loaded_model, database, db_neighbours)
            print(img_paths)
            for i in range(len(img_paths)):
                st.image(PIL.Image.open('C:'+img_paths[i][0]),
                         caption='Image {} with dist {}'.format(i + 1, f'{dists[i]:.3f}', width=580))
        except Exception as e:
            st.write('CRASHED:{}'.format(e))

# main()

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from cauth.models import UserProfile
from BetApp.models import Bet
from forms import BetForm
from random import randint
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required

# Create your views here.

def index(request):
	return render(request, 'index.html', {'home_active': True})

@login_required
def balance(request):
	profile = UserProfile.objects.get(user_id=request.user.id)
	if request.method == "POST":
		input_balance = int(request.POST.get('balance'))
		if input_balance > 50:
			input_balance += input_balance * 0.10
		profile.initial_balance += input_balance
		profile.save()
	balance = int(profile.initial_balance)
	return render(request, 'balance.html', {'balance': balance, 'balance_active': True})

@login_required
def bet(request):
	profile = UserProfile.objects.get(user_id=request.user.id)
	balance = int(profile.initial_balance)
	if request.method == "POST":
		form = BetForm(request.POST)
		if form.is_valid:
			bet = int(request.POST.get('bet'))
			choice = 1
			if bet > balance:
				return render(request, 'bet.html', {'bet_active': True, 'balance': balance, 'errors': True, 'error': 'You do not have suficient credits.'})
			elif bet == 0 or bet > 20:
				return render(request, 'bet.html', {'bet_active': True, 'balance': balance, 'errors': True, 'error': 'You must bet 1 - 20.'})
			else:
				# --------- bet logic
				# 0 -> win  1 -> lose
				choice = randint(0,1)
				win = bet + (20/bet) * 100
				betDB = Bet()
				betDB.bet = bet
				betDB.choice = choice
				betDB.winnings = win
				betDB.user = request.user
				betDB.save()

				# Update user balance
				profile.initial_balance -= bet

				if choice == 0:
					profile.initial_balance += win
					profile.save()
					balance = int(profile.initial_balance)
					return render(request, 'bet.html', {'bet_active': True, 'balance': balance, 'play': True, 'win': win, 'winner': True})
				else:
					profile.save()
					balance = int(profile.initial_balance)
					return render(request, 'bet.html', {'bet_active': True, 'balance': balance, 'play': True, 'win': 0, 'winner': False})
	else:
		return render(request, 'bet.html', {'bet_active': True, 'balance': balance, 'play': False})

@login_required()
def bet_history(request):
	args = {}
	Bets = Bet.objects.filter(user_id=request.user.id)
	args['Bets'] = Bets
	return render(request, 'bet_history.html', args)


import numpy as np
import random
from scipy.signal import convolve2d

class Minesweeper:
    """
    A Minesweeper game engine.

    Attributes:
        shape (tuple): The shape of the game board.
        num_mines (int): Total number of mines on the board.
        board (np.ndarray): A matrix representing the game board.
        revealed (np.ndarray): A matrix representing revealed cells.
        mines (set): A set containing coordinates of mines.
        game_over (bool): Indicates if the game is over.
        game_won (bool): Indicates if the game is won.
        num_revealed (int): Number of revealed cells.
        num_flags (int): Number of flagged cells.
        zeros (np.ndarray): A matrix identifying zeros on the board.
    """

    def __init__(self, shape, num_mines):
        """
        Initializes a new Minesweeper game.

        Args:
            shape (tuple): Shape of the game board (rows, columns).
            num_mines (int): Number of mines to be placed on the board.
        """
        self.shape = shape
        self.num_mines = num_mines
        self.board = np.zeros(shape, dtype=int)
        self.revealed = np.zeros(shape, dtype=int)
        self.mines = set()
        self.game_over = False
        self.game_won = False
        self.num_revealed = 0
        self.num_flags = 0
        self.initialize_board()
        self.zeros = np.where(self.board == 0, 1, 0)

    def initialize_board(self):
        """Initializes the board with mines and computes the numbers for surrounding cells."""
        indices = [(i, j) for i in range(self.shape[0]) for j in range(self.shape[1])]
        mine_indices = random.sample(indices, self.num_mines)
        for i, j in mine_indices:
            self.board[i][j] = -1
            self.mines.add((i, j))
            for ii in range(i-1, i+2):
                for jj in range(j-1, j+2):
                    if ii >= 0 and ii < self.shape[0] and jj >= 0 and jj < self.shape[1] and self.board[ii][jj] != -1:
                        self.board[ii][jj] += 1

    def reveal(self, i, j):
        """
        Reveals a cell and its surroundings if it's an empty cell.

        Args:
            i (int): Row index of the cell to reveal.
            j (int): Column index of the cell to reveal.
        """
        if (i, j) in self.mines:
            self.game_over = True
            return

        if self.board[i][j] != 0:
            self.revealed[i][j] = True
            return

        queue = [(i, j)]
        queue_discovered = []

        while queue:
            (i,j) = queue.pop(0)
            for ii in range(max(i-1, 0), min(i+1, self.shape[0]-1) + 1):
                for jj in range(max(j-1, 0), min(j+1, self.shape[1]-1) + 1):
                    if self.board[ii,jj] >= 0 and self.board[ii,jj] <= 8 and (ii,jj) not in queue and (ii,jj) not in queue_discovered:
                        if self.board[ii,jj] == 0:
                            queue.append((ii,jj))
                        self.revealed[ii][jj] = True
            queue_discovered.append((i,j))

    def reveal_random (self):
        """Reveals a random cell that has a value of 0."""
        if self.game_over or self.game_won:
            self.game_over = True
            return

        indicies = []
        for i in range(self.shape[0]):
            for j in range(self.shape[1]):
                if self.board[i][j] == 0:
                    indicies.append((i,j))

        random_index = random.choice(indicies)

        i,j = random_index

        # print(i,j)

        if self.game_over or self.game_won or (i, j) in self.mines:
            self.game_over = True
            return

        self.reveal(i,j)
        return random_index

    def reveal_random_more(self, num_reveals):
        """
        Reveals multiple random border cells.

        Args:
            num_reveals (int): Number of random border cells to reveal.
        """
        random_index = None
        for i in range(num_reveals):

            indicies = []
            for i in range(self.shape[0]):
                for j in range(self.shape[1]):
                    if not (self.board[i][j] in [-3, -1]) and self.get_border()[i][j] == 1:
                        indicies.append((i,j))
            if indicies == []:
                return
            random_index = random.choice(indicies)

            i,j = random_index

            # print(i,j)

            if self.game_over or self.game_won or (i, j) in self.mines:
                self.game_over = True
                return

            self.reveal(i,j)

        return random_index

    def flag(self, i, j):
        """
        Flags a cell.

        Args:
            i (int): Row index of the cell to flag.
            j (int): Column index of the cell to flag.
        """
        if self.board[i][j] >= 0:
            self.board[i][j] = -3
            self.num_flags += 1

    def unflag(self, i, j):
        """
        Unflags a cell.

        Args:
            i (int): Row index of the cell to unflag.
            j (int): Column index of the cell to unflag.
        """
        if self.board[i][j] == -3:
            self.board[i][j] = 0
            self.num_flags -= 1

    def get_game_status(self):
        """
        Gets the current game status.

        Returns:
            str: "lost" if game is lost, "won" if game is won, "ongoing" otherwise.
        """
        if self.game_over:
            return "lost"
        elif self.game_won:
            return "won"
        else:
            return "ongoing"

    def print_board(self):
        """Prints the current state of the game board."""
        for i in range(self.shape[0]):
            for j in range(self.shape[1]):
                if self.board[i][j] == -2:
                    print(".", end=" ")
                elif self.board[i][j] == -3:
                    print("F", end=" ")
                else:
                    print(" ", end=" ")
            print()

    def get_board(self):
        """Returns the current state of the game board."""
        return self.board

    def get_revealed(self):
        """Returns the current state of revealed cells."""
        return self.revealed

    def get_border(self):
        """
        Computes the border cells that are adjacent to revealed cells.

        Returns:
            np.ndarray: A matrix indicating border cells.
        """
        kernel = np.ones((3, 3))
        convolved = convolve2d(self.revealed, kernel, mode='same')
        result = np.logical_and(convolved > 0, self.revealed == 0).astype(int)
        return result

# The values of the board are as follows:
# -1: mine
# -2: revealed
# -3: flagged
# -4: hidden
# 0-8: number of mines in the surrounding cells

# Minesweeper class documentation for someone to use it in their code
# Minesweeper(shape, num_mines): creates a Minesweeper object with the given shape and number of mines
# reveal(i, j): reveals the cell at (i, j) and all the surrounding cells if the cell is empty
# flag(i, j): flags the cell at (i, j)
# unflag(i, j): unflags the cell at (i, j)
# get_board(): returns the current board
# get_game_status(): returns the current game status
# print_board(): prints the current board


# madLibs.py - program that reads text file and let user add their own text anywhere the word
# ADJECTIVE, NOUN, ADVERB, or VERB appears in the text file.
from pathlib import Path
import time
import re
import pyinputplus as pyip
import time

# This creates a path to folder separated for this project
filesFolder = (Path.cwd() / 'chapter_09\\madLibsTexts')

# Regexes created to found given word
# and to found if currently checked word ends with a . , or some other sign.
libs_regex = re.compile(r'^ADJECTIVE|^NOUN|^ADVERB|^VERB')
letter_regex = re.compile(r'\W')

template_file = open(f'{str(filesFolder)}\\basic_text.txt', 'r')
template_file_content = template_file.read()
list_of_separated_words = template_file_content.split()

for index, word in enumerate(list_of_separated_words):
    matching_word = libs_regex.search(word)

    # If given word is found, it prompts user to type in word the user sees as correct.
    if matching_word != None:
        user_input = pyip.inputStr(prompt='Enter ' + matching_word.group().lower() + ':\n')

        # Checks if a word ends with a . or , or some other sign.
        # If yes, then puts it at the end of the new word.
        if letter_regex.search(word[-1]):
            user_input = user_input + word[-1]

        list_of_separated_words[index] = user_input

new_string = ' '.join(list_of_separated_words)

# timestamp generated for file name.
timestamp_note = int(time.time())

# New file is created. To differ between new files, timestamp is added to its name.
end_file = open(f'{str(filesFolder)}\\filled_text{timestamp_note}.txt', 'w')
end_file.write(new_string)

template_file.close()
end_file.close()

# find each occurence of said words - regex? ^ADJECTIVE | ^N
# for each occurence, show input prompt so it can be chanded
# print the whole thing
# save it to new text file


from django.shortcuts import render

# Create your views here.

import requests

def index(request):
    # https://openweathermap.org/  --Create an account here to get the api key
    # url = 'http://api.openweathermap.org/data/2.5/weather?q={}&units=imperial&appid=API_KEY' -- we have to give the API_KEY
    url = 'http://api.openweathermap.org/data/2.5/weather?q={}&units=imperial&appid=f0d345e3321ea2ada0be02c622db15b0'

    city1="Kottayam"
    city1 = requests.get(url.format(city1)).json() #we are requesting the API data and converting the JSON to Python data types
    weather1 = {
        'city' : city1['name'],
        'temperature' : city1['main']['temp'],
        'description' : city1['weather'][0]['description'],
        'wind' : city1['wind']['speed'],
        'icon' : city1['weather'][0]['icon'],
        'country' : city1['sys']['country']
    }

    city2 = 'Pune'
    city2 = requests.get(url.format(city2)).json() #we are requesting the API data and converting the JSON to Python data types
    weather2 = {
        'city' : city2['name'],
        'temperature' : city2['main']['temp'],
        'description' : city2['weather'][0]['description'],
        'wind' : city2['wind']['speed'],
        'icon' : city2['weather'][0]['icon'],
        'country' : city2['sys']['country']
    }

    return render(request, 'index.html', {'weather1' : weather1,'weather2':weather2}) #returns the index.html template


import testtools

# Third Party Imports

# Cloudify Imports is imported and used in operations
from cloudify.exceptions import NonRecoverableError
from docker_plugin import docker_client

class TestClient(testtools.TestCase):

    def test_bad_daemon_dictionary(self):
        daemon_client = {
            'base_url': 'ZZZZZZXXXXXX'
        }
        ex = self.assertRaises(
            NonRecoverableError, docker_client.get_client, daemon_client)
        self.assertIn('Error while getting client', ex.message)


import streamlit as st
from namedivider import BasicNameDivider, GBDTNameDivider
from pathlib import Path
Path("~/.cache").expanduser().mkdir(exist_ok=True)

@st.experimental_singleton
def get_basic_name_divider():
    basic_divider = BasicNameDivider()
    return basic_divider

@st.experimental_singleton
def get_gbdt_name_divider():
    gbdt_divider = GBDTNameDivider()
    return gbdt_divider

def main():
    col1, col2, col3, col4, col5 = st.columns(5)
    with col1:
        option = st.selectbox(
            'Language',
            ("Japanese", 'English'))

    basic_divider = get_basic_name_divider()
    gbdt_divider = get_gbdt_name_divider()

    if option == "Japanese":
        st.title("NameDivider Demo")
        st.write("NameDividerは、姓名が連結している日本語の名前を姓と名に分割するツールです。")
        st.write("https://github.com/rskmoi/namedivider-python")
        st.write("左側のテキストエリアに名前を入力すると、右側に分割された名前が出力されます。")
        mode_select = "分割手法を選択してください"
        mode_basic = "BasicNameDivider(速いけれど、精度が99.2%)"
        mode_gbdt = "GBDTNameDivider(遅いけれど、精度が99.9%)"
        input_label = "入力"
        output_label = "結果"
        too_many_error = "一度に分割できる名前の数は500人までです"
        info = 'このページはNameDividerの使用感をお伝えするためのデモページです。\n ' \
            '1. このページの使用により生じる不利益について作者は一切責任を負わないものとします。 \n ' \
            '2. このページは任意のタイミングで非公開になる可能性が有ります。 \n ' \
            '3. 入力された名前は保存をしておらず、作者が確認することはできません。'
    elif option == "English":
        st.title("NameDivider Demo")
        st.write("NameDivider is a tool for dividing the Japanese full name into a family name and a given name.")
        st.write("https://github.com/rskmoi/namedivider-python")
        st.write("Entering names in the text area on the left side will output the divided names on the right side.")
        mode_select = "Please select a division method."
        mode_basic = "BasicNameDivider(Fast, but accuracy is 99.2%)"
        mode_gbdt = "GBDTNameDivider(Slow, but accuracy is 99.9%)"
        input_label = "Input undivided names."
        output_label = "Divided names are shown here."
        too_many_error = "The number of names that can be divided at one time is limited to 500"
        info = 'This page is a demo page to show how NameDivider works.\n ' \
            '1. The author assumes no responsibility for any disadvantages that may result from the use of this page. \n ' \
            '2. This page may become private at any time. \n ' \
            '3. The name entered is not saved and cannot be confirmed by the author.'

    else:
        raise ValueError

    mode = st.radio(
        mode_select,
        (mode_basic, mode_gbdt))

    input_area, output_area = st.columns(2)
    undivided_names = input_area.text_area(input_label,
                                           height=500,
                                           placeholder="菅義偉\n岸田文雄\n鳩山由紀夫")
    divided_names = []
    undivided_names_list = undivided_names.split("\n")
    if len(undivided_names_list) > 500:
        st.error(too_many_error)
    for _name in undivided_names_list:
        if len(_name) < 2:
            continue
        if mode == mode_basic:
            _divided_name = str(basic_divider.divide_name(_name))
        elif mode == mode_gbdt:
            _divided_name = str(gbdt_divider.divide_name(_name))
        else:
            raise ValueError
        divided_names.append(_divided_name)
    output_area.text_area(label=output_label,
                          value="\n".join(divided_names),
                          height=500,
                          placeholder="菅 義偉\n岸田 文雄\n鳩山 由紀夫")

    st.info(info)

if __name__ == '__main__':
    main()


import argparse
import os
import pickle
import sys

import numpy as np
import seaborn as sns

from pysrc import plots
from pysrc.sampling import baseline
from pysrc.services.data_service import load_site_data
from pysrc.services.file_service import (
    logs_dir_path,
    output_dir_path,
    plots_dir_path,
)

sys.path.append(os.path.abspath("src"))
sns.set(font_scale=1.2)

# Read arguments from stdin
parser = argparse.ArgumentParser(description="parameter settings")

parser.add_argument("--model", type=str, default="full_model_v3")
parser.add_argument("--xi", type=float, default=2.0)
parser.add_argument("--pf", type=float, default=21.5)
parser.add_argument("--pa", type=float, default=42.03)
parser.add_argument("--weight", type=float, default=0.25)
parser.add_argument("--sitenum", type=int, default=78)
parser.add_argument("--timehzn", type=int, default=200)

# Parse arguments
args = parser.parse_args()

# Create output and plots directories
output_dir = output_dir_path(**vars(args))
plots_dir = plots_dir_path(**vars(args))
logs_dir = logs_dir_path(**vars(args))

with open(output_dir / "results.pcl", "rb") as f:
    # Load the data from the file
    results = pickle.load(f)

# Load site data
(
    zbar_2017,
    gamma,
    z_2017,
    forest_area_2017,
    theta,
    site_theta_2017_df,
    site_gamma_2017_df,
    municipal_theta_df,
    municipal_gamma_df,
) = load_site_data(args.sitenum)

# Load coef baseline samples
fit = baseline.sample(model_name=args.model, num_samples=10000, num_sites=args.sitenum)
base_samples = np.concatenate(
    (fit.stan_variable("theta"), fit.stan_variable("gamma")), axis=1
)

try:
    adj_samples = results["final_sample"]
except KeyError:
    print(
        """
        Algorithm did not finish converging.
        Plotting adjusted samples from last iteration...
        """
    )
    adj_samples = results["collected_ensembles"][results["cntr"] - 1]

# Plot overlapped densities
plots.density_overlap(base_samples, adj_samples, plots_dir, args.sitenum)

# Plot absolute and percentage error
plots.traceplot_abs_error(results, plots_dir)
plots.traceplot_pct_error(results, plots_dir)

# Plot sampling time
plots.traceplot_sampling_time(results, plots_dir)

# Plot theta and gamma convergence
plots.traceplot_params_pct_error(results, plots_dir)
plots.traceplot_gammas(results, plots_dir)
plots.traceplot_thetas(results, plots_dir)

# Plot agg %change in z
plots.agg_Z_trajectory(
    z_2017=z_2017,
    zbar_2017=zbar_2017,
    results=results,
    num_sites=args.sitenum,
    plots_dir=plots_dir,
)

# Plot Z (for each site)
plots.Z_trajectory(results, plots_dir)

# Plot X (aggregate among sites)
plots.X_trajectory(results, plots_dir)

plots.delta_Z_trajectory(results, plots_dir)

# Plot adjustments
plots.adj_costs_trajectory(results, plots_dir)

# Plot trajectory of controls
plots.V_trajectory(results, plots_dir)
plots.U_trajectory(results, plots_dir)


from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.feature_selection import SelectKBest, mutual_info_classif
from sklearn.model_selection import train_test_split
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.metrics import confusion_matrix
import pandas as pd
from sklearn.metrics import classification_report

unclean_df = pd.read_csv(r'C:\Users\User\Desktop\Applied Research Project\HeartDiseasePrediction (8) (1)\HeartDiseasePrediction\HeartDiseasePrediction\HeartDiseasePrediction\heart.csv')

unclean_df = unclean_df.drop_duplicates(subset=None, keep='first', inplace=False, ignore_index=False)
unclean_df['age'] = unclean_df['age'].apply(lambda x: round(x))

unclean_df['age'] = unclean_df['age'].apply(lambda x: round(x))

unclean_df.to_csv('cleaned_data2.csv')
df = unclean_df
original_data = df
X = df.drop(columns=['target'])
y = df['target']
SelectKBest_features = SelectKBest(mutual_info_classif, ).fit(X, y)

SelectKBest_features.get_support(indices=True)
X_method3 = X.iloc[:, SelectKBest_features.get_support(indices=True)]
X_method3

X_train3, X_test3, y_train3, y_test3 = train_test_split(X_method3, y, test_size=0.3, random_state=42)
logreg3 = LogisticRegression(max_iter=5000)
logreg3.fit(X_train3, y_train3)
y_pred_3 = logreg3.predict(X_test3)
print('Accuracy of logistic regression classifier on test set: {:.6f}'.format(logreg3.score(X_test3, y_test3)))
new_features = X_method3
new_features
ss = StandardScaler()

zscore = ss.fit_transform(new_features)

X_scaled = pd.DataFrame(zscore, index=new_features.index, columns=new_features.columns)
X_scaled = X_scaled.reset_index(drop=True)
X_scaled

X_train1, X_test1, y_train1, y_test1 = train_test_split(X_scaled, y, test_size=0.25, random_state=0)

X_train = X_train1
y_train = y_train1
X_test = X_test1
y_test = y_test1
AdaBoost_Classifier = AdaBoostClassifier(random_state=0).fit(X_train1, y_train1)
ada_scores = cross_val_score(AdaBoost_Classifier, X_train, y_train, cv=3)
ada_scores_max = ada_scores.max()
ada_scores_max

AdaBoost_Classifier_pred = AdaBoost_Classifier.predict(X_test1)

AdaBoost_Classifier_acc = accuracy_score(y_test1, AdaBoost_Classifier_pred)
AdaBoost_Classifier_acc
AdaBoost_Classifier_matrix = confusion_matrix(y_test1, AdaBoost_Classifier_pred)
AdaBoost_Classifier_matrix

print(classification_report(y_test1, AdaBoost_Classifier_pred))


import commands
import logging
import ce_deploy_scripts

def get_host_conn_state(hosts):

    for key in hosts:
        logging.info('\ncheck connection with %s\n' % key)
        shell_file_path = ce_deploy_scripts.BASE_DIR + "/shell_script/netping.sh"
        shell_cmd = "sh " + shell_file_path + " " + hosts[key]
        output = commands.getstatusoutput(shell_cmd)
        logging.info('\n%s\n' % output[1])
        if output[0] == 0:
            logging.error('\nhost %s is unavailable\n' % key)
            return False

    return True


import os
import redis.asyncio as redis
from pyrogram.client import Client
from pyrogram import filters

redis_db = redis.from_url("redis://redis")

api_id = os.getenv("API_ID")
api_hash = os.getenv("API_HASH")
app = Client("bayanist", api_id=api_id, api_hash=api_hash)

CHANNELS_TO_LISTEN = set([int(chat_id) for chat_id in os.getenv("CHANNELS_TO_LISTEN").split(",")])
MAX_QUEUE_CAPACITY = 250
QUEUE_NAME = "memes_queue"

def specific_channel_filter(_, __, message):
    return message.chat.id in CHANNELS_TO_LISTEN

is_chosen_channel = filters.create(specific_channel_filter)

async def add_meme_to_queue(data):
    if await redis_db.llen(QUEUE_NAME) >= MAX_QUEUE_CAPACITY:
        await redis_db.rpop(QUEUE_NAME)
    await redis_db.lpush(QUEUE_NAME, data)

@app.on_message(filters.channel & ~filters.forwarded & is_chosen_channel & filters.photo)
async def await_memes(client, message):
    f = await client.download_media(message.photo, in_memory=True)
    await add_meme_to_queue(f.getvalue())

if __name__ == "__main__":
    app.run()


# -*- coding: utf-8 -*-

# $Id$

from __future__ import unicode_literals
from builtins import object
from future.utils import PY2

import os
import re
import shutil
import tempfile
import unittest

import pyinotify

from arv.autotest.config import watch_node_validator
from arv.autotest.event_filters import and_
from arv.autotest.event_filters import is_delete_dir_event
from arv.autotest.event_filters import not_
from arv.autotest.event_filters import simple_event_filter_factory
from arv.autotest.event_filters import throttler_factory

class Object(object):
    def __init__(self, **kw):
        self.__dict__.update(kw)

def make_event(path, name, mask=0):
    return Object(path=path, name=name, mask=mask)

def make_watch(path, recurse=False, auto_add=False, include=[], exclude=[]):
    return watch_node_validator({
        "path": path,
        "recurse": recurse,
        "auto_add": auto_add,
        "include": include,
        "exclude": exclude
    })

def make_fake_timer(start, deltas):
    def generator():
        total = start
        yield total
        for i in deltas:
            total += i
            yield total
    if PY2:
        return generator().next
    else:
        return generator().__next__

class TestIsDirEvent(unittest.TestCase):

    def test_exclude_directory_delete(self):
        event = make_event("", "", pyinotify.IN_DELETE | pyinotify.IN_ISDIR)
        self.assert_(is_delete_dir_event(event))

    def test_exclude_self_delete(self):
        event = make_event("", "", pyinotify.IN_DELETE | pyinotify.IN_DELETE_SELF)
        self.assert_(is_delete_dir_event(event))

    def test_include_directory_create(self):
        event = make_event("", "", pyinotify.IN_CREATE | pyinotify.IN_ISDIR)
        self.failIf(is_delete_dir_event(event))

class TestSimpleEventFilterFactory(unittest.TestCase):

    def setUp(self):
        self.tmpdir = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmpdir)

    def make_subdir(self, name):
        name = os.path.join(self.tmpdir, name)
        os.mkdir(name)
        return name

    def test_exclude_excluded_file(self):
        event = make_event(self.tmpdir, "exclude.abc")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, exclude=[u".*\\.abc"])
        ])
        self.failIf(filter(event))

    def test_include_included_file(self):
        event = make_event(self.tmpdir, "include.abc")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, include=[u".*\\.abc"])
        ])
        self.assert_(filter(event))

    def test_exclude_excluced_file_in_subdirectory(self):
        subdir = self.make_subdir("subdir")
        event = make_event(subdir, "exclude.abc")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, exclude=[u".*\\.abc"])
        ])
        self.failIf(filter(event))

    def test_include_included_file_in_subdirectory(self):
        subdir = self.make_subdir("subdir")
        event = make_event(subdir, "include.abc")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, include=[u".*\\.abc"])
        ])
        self.assert_(filter(event))

    def test_exclude_specific_takes_over_generic(self):
        subdir = self.make_subdir("subdir")
        event = make_event(subdir, "exclude.abc")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, include=[u".*\\.abc"]),  # /tmp
            make_watch(subdir, exclude=[u".*\\.abc"]),       # /tmp/subdir
        ])
        self.failIf(filter(event))

    def test_include_specific_takes_over_generic(self):
        subdir = self.make_subdir("subdir")
        event = make_event(subdir, "exclude.abc")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, exclude=[u".*\\.abc"]),  # /tmp
            make_watch(subdir, include=[u".*\\.abc"]),       # /tmp/subdir
        ])
        self.assert_(filter(event))

    def test_global_exclude_takes_over_all(self):
        event = make_event(self.tmpdir, ".exclude.abc")
        filter = simple_event_filter_factory(
            watches=[
                make_watch(self.tmpdir, include=[u".*\\.abc"]),
            ],
            global_ignores=[re.compile("\\..*")]
        )
        self.failIf(filter(event))

    def test_neither_excluded_nor_included_gets_excluded_by_default(self):
        event = make_event(self.tmpdir, "exclude.txt")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, exclude=[u".*\\.abc"], include=[u".*\\.py"])
        ])
        self.failIf(filter(event))

    def test_unwatched_directory_raises_ValueError(self):
        parent = os.path.dirname(self.tmpdir)
        event = make_event(parent, "exclude.abc")
        filter = simple_event_filter_factory([
            make_watch(self.tmpdir, include=[u".*\\.abc"])
        ])
        self.assertRaises(ValueError, filter, event)

class TestFilterCombinators(unittest.TestCase):

    def setUp(self):
        def is_odd(value):
            return bool(value % 2)

        def is_big(value):
            return value > 100

        self.is_odd = is_odd
        self.is_big = is_big

    def test_preconditions(self):
        self.assert_(self.is_odd(3))
        self.failIf(self.is_odd(2))
        self.assert_(self.is_big(1000))
        self.failIf(self.is_big(2))

    def test_negation(self):
        f = not_(self.is_odd)
        self.assert_(f(2))
        self.failIf(f(3))

    def test_and(self):
        f = and_(self.is_odd, self.is_big)
        self.assert_(f(1003))
        self.failIf(f(1002))
        self.failIf(f(3))
        self.failIf(f(2))

class TestThrottlingFilter(unittest.TestCase):

    def set_up(self, deltas, start=0, count=10):
        self.cfg = Object(max_events_second=count)  # 10 events/second
        self.timer = make_fake_timer(start, deltas)
        self.throttler = throttler_factory(self.cfg, self.timer)
        self.event = make_event("/tmp", "foo")

    def test_fake_timer(self):
        timer = make_fake_timer(5, [0.1, 0, 0.5])
        self.assertEqual(timer(), 5)
        self.assertEqual(timer(), 5.1)
        self.assertEqual(timer(), 5.1)
        self.assertEqual(timer(), 5.6)
        self.assertRaises(StopIteration, timer)

    def test_first_event_is_accepted(self):
        self.set_up([])
        self.assert_(self.throttler(self.event))

    def test_event_accepted_if_delta_elapsed(self):
        self.set_up([0.1])
        self.throttler(self.event)
        self.assert_(self.throttler(self.event))

    def test_event_accepted_if_delta_elapsed_cummulative(self):
        self.set_up([0.05, 0.06])
        self.throttler(self.event)
        self.failIf(self.throttler(self.event))
        self.assert_(self.throttler(self.event))

    def test_event_rejected_if_delta_not_elapsed(self):
        self.set_up([0.01])
        self.throttler(self.event)
        self.failIf(self.throttler(self.event))

    def test_None_cfg_disables_throttling(self):
        throttler = throttler_factory(None, make_fake_timer(0, [0, 0]))
        event = make_event("/tmp", "foo")
        self.assert_(throttler(event))  # always accepted
        self.assert_(throttler(event))  # two deltas
        self.assert_(throttler(event))
        self.assert_(throttler(event))  # timer isn't called

    def test_adjust_delta(self):
        self.set_up([0.2, 0.5])
        self.throttler(self.event)
        self.assert_(self.throttler(self.event))
        self.throttler.adjust_delta(0.7)
        self.failIf(self.throttler(self.event))