# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.6.0
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %% [markdown]
# # Workflow for a multi-regional energy system
#
# In this application of the FINE framework, a multi-regional energy system is modeled and optimized.
#
# All classes which are available to the user are utilized and examples of the selection of different parameters within these classes are given.
#
# The workflow is structures as follows:
# 1. Required packages are imported and the input data path is set
# 2. An energy system model instance is created
# 3. Commodity sources are added to the energy system model
# 4. Commodity conversion components are added to the energy system model
# 5. Commodity storages are added to the energy system model
# 6. Commodity transmission components are added to the energy system model
# 7. Commodity sinks are added to the energy system model
# 8. The energy system model is optimized
# 9. Selected optimization results are presented
#

# %% [markdown]
# # 1. Import required packages and set input data path
#
# The FINE framework is imported which provides the required classes and functions for modeling the energy system.

# %%
import FINE as fn
import matplotlib.pyplot as plt
from getData import getData
import pandas as pd
import os

cwd = os.getcwd()
data = getData()

# %matplotlib inline
# %load_ext autoreload
# %autoreload 2

# %% [markdown]
# # 2. Create an energy system model instance
#
# The structure of the energy system model is given by the considered locations, commodities, the number of time steps as well as the hours per time step.
#
# The commodities are specified by a unit (i.e. 'GW_electric', 'GW_H2lowerHeatingValue', 'Mio. t CO2/h') which can be given as an energy or mass unit per hour. Furthermore, the cost unit and length unit are specified.

# %%
locations = {'cluster_0', 'cluster_1', 'cluster_2', 'cluster_3', 'cluster_4', 'cluster_5', 'cluster_6', 'cluster_7'}
commodityUnitDict = {'electricity': r'GW$_{el}$', 'methane': r'GW$_{CH_{4},LHV}$', 'biogas': r'GW$_{biogas,LHV}$',
                     'CO2': r'Mio. t$_{CO_2}$/h', 'hydrogen': r'GW$_{H_{2},LHV}$'}
commodities = {'electricity', 'hydrogen', 'methane', 'biogas', 'CO2'}
numberOfTimeSteps=8760
hoursPerTimeStep=1

# %%
esM = fn.EnergySystemModel(locations=locations, commodities=commodities, numberOfTimeSteps=8760,
                           commodityUnitsDict=commodityUnitDict,
                           hoursPerTimeStep=1, costUnit='1e9 Euro', lengthUnit='km', verboseLogLevel=0)

# %%
CO2_reductionTarget = 1

# %% [markdown]
# # 3. Add commodity sources to the energy system model

# %% [markdown]
# ## 3.1. Electricity sources

# %% [markdown]
# ### Wind onshore

# %%
esM.add(fn.Source(esM=esM, name='Wind (onshore)', commodity='electricity', hasCapacityVariable=True,
                  operationRateMax=data['Wind (onshore), operationRateMax'],
                  capacityMax=data['Wind (onshore), capacityMax'],
                  investPerCapacity=1.1, opexPerCapacity=1.1*0.02, interestRate=0.08,
                  economicLifetime=20))

# %% [markdown]
# Full load hours:

# %%
data['Wind (onshore), operationRateMax'].sum()

# %% [markdown]
# ### Wind offshore

# %%
esM.add(fn.Source(esM=esM, name='Wind (offshore)', commodity='electricity', hasCapacityVariable=True,
                  operationRateMax=data['Wind (offshore), operationRateMax'],
                  capacityMax=data['Wind (offshore), capacityMax'],
                  investPerCapacity=2.3, opexPerCapacity=2.3*0.02, interestRate=0.08,
                  economicLifetime=20))

# %% [markdown]
# Full load hours:

# %%
data['Wind (offshore), operationRateMax'].sum()

# %% [markdown]
# ### PV

# %%
esM.add(fn.Source(esM=esM, name='PV', commodity='electricity', hasCapacityVariable=True,
                  operationRateMax=data['PV, operationRateMax'], capacityMax=data['PV, capacityMax'],
                  investPerCapacity=0.65, opexPerCapacity=0.65*0.02, interestRate=0.08,
                  economicLifetime=25))

# %% [markdown]
# Full load hours:

# %%
data['PV, operationRateMax'].sum()

# %% [markdown]
# ### Exisisting run-of-river hydroelectricity plants

# %%
esM.add(fn.Source(esM=esM, name='Existing run-of-river plants', commodity='electricity',
                  hasCapacityVariable=True,
                  operationRateFix=data['Existing run-of-river plants, operationRateFix'], tsaWeight=0.01,
                  capacityFix=data['Existing run-of-river plants, capacityFix'],
                  investPerCapacity=0, opexPerCapacity=0.208))

# %% [markdown]
# ## 3.2. Methane (natural gas and biogas)

# %% [markdown]
# ### Natural gas

# %%
esM.add(fn.Source(esM=esM, name='Natural gas purchase', commodity='methane',
                  hasCapacityVariable=False, commodityCost=0.0331*1e-3))

# %% [markdown]
# ### Biogas

# %%
esM.add(fn.Source(esM=esM, name='Biogas purchase', commodity='biogas',
                  operationRateMax=data['Biogas, operationRateMax'], hasCapacityVariable=False,
                  commodityCost=0.05409*1e-3))

# %% [markdown]
# # 4. Add conversion components to the energy system model

# %% [markdown]
# ### Combined cycle gas turbine plants

# %%
esM.add(fn.Conversion(esM=esM, name='CCGT plants (methane)', physicalUnit=r'GW$_{el}$',
                      commodityConversionFactors={'electricity':1, 'methane':-1/0.6, 'CO2':201*1e-6/0.6},
                      hasCapacityVariable=True,
                      investPerCapacity=0.65, opexPerCapacity=0.021, interestRate=0.08,
                      economicLifetime=33))

# %% [markdown]
# ### New combined cycle gas turbine plants for biogas

# %%
esM.add(fn.Conversion(esM=esM, name='New CCGT plants (biogas)', physicalUnit=r'GW$_{el}$',
                      commodityConversionFactors={'electricity':1, 'biogas':-1/0.63},
                      hasCapacityVariable=True,
                      investPerCapacity=0.7, opexPerCapacity=0.021, interestRate=0.08,
                      economicLifetime=33))

# %% [markdown]
# ### New combined cycly gas turbines for hydrogen

# %%
esM.add(fn.Conversion(esM=esM, name='New CCGT plants (hydrogen)', physicalUnit=r'GW$_{el}$',
                      commodityConversionFactors={'electricity':1, 'hydrogen':-1/0.63},
                      hasCapacityVariable=True,
                      investPerCapacity=0.7, opexPerCapacity=0.021, interestRate=0.08,
                      economicLifetime=33))

# %% [markdown]
# ### Electrolyzers

# %%
esM.add(fn.Conversion(esM=esM, name='Electrolyzer', physicalUnit=r'GW$_{el}$',
                      commodityConversionFactors={'electricity':-1, 'hydrogen':0.7},
                      hasCapacityVariable=True,
                      investPerCapacity=0.5, opexPerCapacity=0.5*0.025, interestRate=0.08,
                      economicLifetime=10))

# %% [markdown]
# ### rSOC

# %%
capexRSOC=1.5

esM.add(fn.Conversion(esM=esM, name='rSOEC', physicalUnit=r'GW$_{el}$', linkedConversionCapacityID='rSOC',
                      commodityConversionFactors={'electricity':-1, 'hydrogen':0.6},
                      hasCapacityVariable=True,
                      investPerCapacity=capexRSOC/2, opexPerCapacity=capexRSOC*0.02/2, interestRate=0.08,
                      economicLifetime=10))

esM.add(fn.Conversion(esM=esM, name='rSOFC', physicalUnit=r'GW$_{el}$', linkedConversionCapacityID='rSOC',
                      commodityConversionFactors={'electricity':1, 'hydrogen':-1/0.6},
                      hasCapacityVariable=True,
                      investPerCapacity=capexRSOC/2, opexPerCapacity=capexRSOC*0.02/2, interestRate=0.08,
                      economicLifetime=10))

# %% [markdown]
# # 5. Add commodity storages to the energy system model

# %% [markdown]
# ## 5.1. Electricity storage

# %% [markdown]
# ### Lithium ion batteries
#
# The self discharge of a lithium ion battery is here described as 3% per month. The self discharge per hours is obtained using the equation (1-$\text{selfDischarge}_\text{hour})^{30*24\text{h}} = 1-\text{selfDischarge}_\text{month}$.

# %%
esM.add(fn.Storage(esM=esM, name='Li-ion batteries', commodity='electricity',
                   hasCapacityVariable=True, chargeEfficiency=0.95,
                   cyclicLifetime=10000, dischargeEfficiency=0.95, selfDischarge=1-(1-0.03)**(1/(30*24)),
                   chargeRate=1, dischargeRate=1, doPreciseTsaModeling=False,
                   investPerCapacity=0.151, opexPerCapacity=0.002, interestRate=0.08,
                   economicLifetime=22))

# %% [markdown]
# ## 5.2. Hydrogen storage

# %% [markdown]
# ### Hydrogen filled salt caverns
# The maximum capacity is here obtained by: dividing the given capacity (which is given for methane) by the lower heating value of methane and then multiplying it with the lower heating value of hydrogen.

# %%
esM.add(fn.Storage(esM=esM, name='Salt caverns (hydrogen)', commodity='hydrogen',
                   hasCapacityVariable=True, capacityVariableDomain='continuous',
                   capacityPerPlantUnit=133,
                   chargeRate=1/470.37, dischargeRate=1/470.37, sharedPotentialID='Existing salt caverns',
                   stateOfChargeMin=0.33, stateOfChargeMax=1, capacityMax=data['Salt caverns (hydrogen), capacityMax'],
                   investPerCapacity=0.00011, opexPerCapacity=0.00057, interestRate=0.08,
                   economicLifetime=30))

# %% [markdown]
# ## 5.3. Methane storage

# %% [markdown]
# ### Methane filled salt caverns

# %%
esM.add(fn.Storage(esM=esM, name='Salt caverns (biogas)', commodity='biogas',
                   hasCapacityVariable=True, capacityVariableDomain='continuous',
                   capacityPerPlantUnit=443,
                   chargeRate=1/470.37, dischargeRate=1/470.37, sharedPotentialID='Existing salt caverns',
                   stateOfChargeMin=0.33, stateOfChargeMax=1, capacityMax=data['Salt caverns (methane), capacityMax'],
                   investPerCapacity=0.00004, opexPerCapacity=0.00001, interestRate=0.08,
                   economicLifetime=30))

# %% [markdown]
# ## 5.4 Pumped hydro storage

# %% [markdown]
# ### Pumped hydro storage

# %%
esM.add(fn.Storage(esM=esM, name='Pumped hydro storage', commodity='electricity',
                   chargeEfficiency=0.88, dischargeEfficiency=0.88,
                   hasCapacityVariable=True, selfDischarge=1-(1-0.00375)**(1/(30*24)),
                   chargeRate=0.16, dischargeRate=0.12, capacityFix=data['Pumped hydro storage, capacityFix'],
                   investPerCapacity=0, opexPerCapacity=0.000153))

# %% [markdown]
# # 6. Add commodity transmission components to the energy system model

# %% [markdown]
# ## 6.1. Electricity transmission

# %% [markdown]
# ### AC cables

# %% [markdown]
# esM.add(fn.LinearOptimalPowerFlow(esM=esM, name='AC cables', commodity='electricity',
#                                   hasCapacityVariable=True, capacityFix=data['AC cables, capacityFix'],
#                                   reactances=data['AC cables, reactances']))

# %%
esM.add(fn.Transmission(esM=esM, name='AC cables', commodity='electricity',
                                  hasCapacityVariable=True, capacityFix=data['AC cables, capacityFix']))

# %% [markdown]
# ### DC cables

# %%
esM.add(fn.Transmission(esM=esM, name='DC cables', commodity='electricity', losses=data['DC cables, losses'],
                        distances=data['DC cables, distances'],
                        hasCapacityVariable=True, capacityFix=data['DC cables, capacityFix']))

# %% [markdown]
# ## 6.2 Methane transmission

# %% [markdown]
# ### Methane pipeline

# %%
esM.add(fn.Transmission(esM=esM, name='Pipelines (biogas)', commodity='biogas',
                        distances=data['Pipelines, distances'],
                        hasCapacityVariable=True, hasIsBuiltBinaryVariable=True, bigM=300,
                        locationalEligibility=data['Pipelines, eligibility'],
                        capacityMax=data['Pipelines, eligibility']*15, sharedPotentialID='pipelines',
                        investPerCapacity=0.000037, investIfBuilt=0.000314,
                        interestRate=0.08, economicLifetime=40))

# %% [markdown]
# ## 6.3 Hydrogen transmission

# %% [markdown]
# ### Hydrogen pipelines

# %%
esM.add(fn.Transmission(esM=esM, name='Pipelines (hydrogen)', commodity='hydrogen',
                        distances=data['Pipelines, distances'],
                        hasCapacityVariable=True, hasIsBuiltBinaryVariable=True, bigM=300,
                        locationalEligibility=data['Pipelines, eligibility'],
                        capacityMax=data['Pipelines, eligibility']*15, sharedPotentialID='pipelines',
                        investPerCapacity=0.000177, investIfBuilt=0.00033,
                        interestRate=0.08, economicLifetime=40))

# %% [markdown]
# # 7. Add commodity sinks to the energy system model

# %% [markdown]
# ## 7.1. Electricity sinks

# %% [markdown]
# ### Electricity demand

# %%
esM.add(fn.Sink(esM=esM, name='Electricity demand', commodity='electricity',
                hasCapacityVariable=False, operationRateFix=data['Electricity demand, operationRateFix']))

# %% [markdown]
# ## 7.2. Hydrogen sinks

# %% [markdown]
# ### Fuel cell electric vehicle (FCEV) demand

# %%
FCEV_penetration=0.5
esM.add(fn.Sink(esM=esM, name='Hydrogen demand', commodity='hydrogen', hasCapacityVariable=False,
                operationRateFix=data['Hydrogen demand, operationRateFix']*FCEV_penetration))

# %% [markdown]
# ## 7.3. CO2 sinks

# %% [markdown]
# ### CO2 exiting the system's boundary

# %%
esM.add(fn.Sink(esM=esM, name='CO2 to enviroment', commodity='CO2',
                hasCapacityVariable=False, commodityLimitID='CO2 limit', yearlyLimit=366*(1-CO2_reductionTarget)))

# %% [markdown]
# # 8. Optimize energy system model

# %% [markdown]
# All components are now added to the model and the model can be optimized. If the computational complexity of the optimization should be reduced, the time series data of the specified components can be clustered before the optimization and the parameter timeSeriesAggregation is set to True in the optimize call.

# %% tags=["nbval-skip"]
esM.cluster(numberOfTypicalPeriods=7)

# %% tags=["nbval-skip"]
esM.optimize(timeSeriesAggregation=True, optimizationSpecs='OptimalityTol=1e-3 method=2 cuts=0')

# %% [markdown]
# # 9. Selected results output
#
# Plot locations (GeoPandas required)

# %% tags=["nbval-skip"]
# Import the geopandas package for plotting the locations
import geopandas as gpd

# %% tags=["nbval-skip"]
locFilePath = os.path.join(cwd, 'InputData', 'SpatialData','ShapeFiles', 'clusteredRegions.shp')
fig, ax = fn.plotLocations(locFilePath, plotLocNames=True, indexColumn='index')

# %% [markdown]
# ### Sources and Sink
#
# Show optimization summary

# %% tags=["nbval-skip"]
esM.getOptimizationSummary("SourceSinkModel", outputLevel=2)

# %% [markdown]
# Plot installed capacities

# %% tags=["nbval-skip"]
fig, ax = fn.plotLocationalColorMap(esM, 'Wind (offshore)', locFilePath, 'index', perArea=False)

# %% tags=["nbval-skip"]
fig, ax = fn.plotLocationalColorMap(esM, 'Wind (onshore)', locFilePath, 'index', perArea=False)

# %% tags=["nbval-skip"]
fig, ax = fn.plotLocationalColorMap(esM, 'PV', locFilePath, 'index', perArea=False)

# %% [markdown]
# Plot operation time series (either one or two dimensional)

# %% tags=["nbval-skip"]
fig, ax = fn.plotOperation(esM, 'Electricity demand', 'cluster_0')

# %% tags=["nbval-skip"]
fig, ax = fn.plotOperationColorMap(esM, 'Electricity demand', 'cluster_0')

# %% [markdown]
# ### Conversion
#
# Show optimization summary

# %% tags=["nbval-skip"]
esM.getOptimizationSummary("ConversionModel", outputLevel=2)

# %% tags=["nbval-skip"]
fig, ax = fn.plotLocationalColorMap(esM, 'Electrolyzer', locFilePath, 'index', perArea=False)

# %% tags=["nbval-skip"]
fig, ax = fn.plotOperationColorMap(esM, 'New CCGT plants (biogas)', 'cluster_2')

# %% [markdown]
# ### Storage
#
# Show optimization summary

# %% tags=["nbval-skip"]
esM.getOptimizationSummary("StorageModel", outputLevel=2)

# %% tags=["nbval-skip"]
fig, ax = fn.plotOperationColorMap(esM, 'Li-ion batteries', 'cluster_2',
                                   variableName='stateOfChargeOperationVariablesOptimum')

# %% tags=["nbval-skip"]
fig, ax = fn.plotOperationColorMap(esM, 'Pumped hydro storage', 'cluster_2',
                                  variableName='stateOfChargeOperationVariablesOptimum')

# %% tags=["nbval-skip"]
fig, ax = fn.plotOperationColorMap(esM, 'Salt caverns (biogas)', 'cluster_2',
                                  variableName='stateOfChargeOperationVariablesOptimum')

# %% tags=["nbval-skip"]
fig, ax = fn.plotOperationColorMap(esM, 'Salt caverns (hydrogen)', 'cluster_2',
                                  variableName='stateOfChargeOperationVariablesOptimum')

# %% [markdown]
# ## Transmission
#
# Show optimization summary

# %% tags=["nbval-skip"]
esM.getOptimizationSummary("TransmissionModel", outputLevel=2).loc['Pipelines (hydrogen)']

# %% [markdown]
# Check that the shared capacity of the pipelines are not exceeded

# %% tags=["nbval-skip"]
df=esM.componentModelingDict["TransmissionModel"].capacityVariablesOptimum
df.loc['Pipelines (biogas)']+df.loc['Pipelines (hydrogen)']

# %% [markdown]
# Plot installed transmission capacities

# %% tags=["nbval-skip"]
transFilePath = os.path.join(cwd, 'InputData', 'SpatialData','ShapeFiles', 'AClines.shp')

fig, ax = fn.plotLocations(locFilePath, indexColumn='index')
fig, ax = fn.plotTransmission(esM, 'AC cables', transFilePath, loc0='bus0', loc1='bus1', fig=fig, ax=ax)

# %% tags=["nbval-skip"]
transFilePath = os.path.join(cwd, 'InputData', 'SpatialData','ShapeFiles', 'DClines.shp')

fig, ax = fn.plotLocations(locFilePath, indexColumn='index')
fig, ax = fn.plotTransmission(esM, 'DC cables', transFilePath, loc0='cluster0', loc1='cluster1', fig=fig, ax=ax)

# %% tags=["nbval-skip"]
transFilePath = os.path.join(cwd, 'InputData', 'SpatialData','ShapeFiles', 'transmissionPipeline.shp')

fig, ax = fn.plotLocations(locFilePath, indexColumn='index')
fig, ax = fn.plotTransmission(esM, 'Pipelines (hydrogen)', transFilePath, loc0='loc1', loc1='loc2',
                              fig=fig, ax=ax)

# %% tags=["nbval-skip"]
transFilePath = os.path.join(cwd, 'InputData', 'SpatialData','ShapeFiles', 'transmissionPipeline.shp')

fig, ax = fn.plotLocations(locFilePath, indexColumn='index')
fig, ax = fn.plotTransmission(esM, 'Pipelines (biogas)', transFilePath, loc0='loc1', loc1='loc2',
                              fig=fig, ax=ax)

# %% [markdown]
# # Postprocessing: Determine robust pipeline design

# %% tags=["nbval-skip"]
# Import module expansion "robustPipelineSizing"
from FINE.expansionModules import robustPipelineSizing

# %% tags=["nbval-skip"]
# 1. Option to get the injection and withdrawal rates for the pipeline sizing (in kg/s)
rates = robustPipelineSizing.getInjectionWithdrawalRates(componentName='Pipelines (hydrogen)',esM=esM) # in GWh
# Convert GWh to kg/s: GWh * (kWh/GWh) * (kg/kWh) * (1/ timestepLengthInSeconds) with timestepLengthInSeconds
# being 3600 seconds for the present example
rates = rates * (10 ** 6) * (1/33.32) * (1/3600)
rates.head()

# %% tags=["nbval-skip"]
# 2. Option to get the injection and withdrawal rates for the pipeline sizing (in kg/s)
op = esM.componentModelingDict[esM.componentNames['Pipelines (hydrogen)']].\
    getOptimalValues('operationVariablesOptimum')['values'].loc['Pipelines (hydrogen)']
rates = robustPipelineSizing.getInjectionWithdrawalRates(operationVariablesOptimumData=op) # in GWh
# Convert GWh to kg/s: GWh * (kWh/GWh) * (kg/kWh) * (1/ timestepLengthInSeconds) with timestepLengthInSeconds
# being 3600 seconds for the present example
rates = rates * (10 ** 6) * (1/33.32) * (1/3600)
rates.head()

# %% tags=["nbval-skip"]
# Determine unique withdrawal and injection scenarios to save computation time
rates = rates.drop_duplicates()
rates.head()

# %% tags=["nbval-skip"]
# Get the lengths of the pipeline (in m)
lengths = robustPipelineSizing.getNetworkLengthsFromESM('Pipelines (hydrogen)', esM)
lengths = lengths * 1e3
lengths.head()

# %% tags=["nbval-skip"]
# Specify minimum and maximum pressure levels for all injection and withdrawal nodes (in bar)
p_min_nodes = {'cluster_5': 60, 'cluster_3': 50, 'cluster_7': 50, 'cluster_1': 60, 'cluster_6': 50,
               'cluster_4': 50, 'cluster_0': 50, 'cluster_2': 50}

p_max_nodes = {'cluster_5': 100, 'cluster_3': 100, 'cluster_7': 100, 'cluster_1': 90, 'cluster_6': 100,
               'cluster_4': 100, 'cluster_0': 100, 'cluster_2': 100}

# %% tags=["nbval-skip"]
# Specify the investment cost of the available diameter classes in €/m

# For single pipes
dic_diameter_costs = {0.1063: 37.51, 0.1307: 38.45, 0.1593: 39.64, 0.2065: 42.12,
                      0.2588: 45.26, 0.3063: 48.69, 0.3356: 51.07, 0.3844: 55.24,
                      0.432: 59.86, 0.4796: 64.98, 0.527: 70.56, 0.578: 76.61,
                      0.625: 82.99, 0.671: 89.95, 0.722: 97.38, 0.7686: 105.28,
                      0.814: 113.63, 0.864: 122.28, 0.915: 131.56, 0.96: 141.3,
                      1.011: 151.5, 1.058: 162.17, 1.104: 173.08, 1.155: 184.67,
                      1.249: 209.24, 1.342: 235.4, 1.444: 263.66, 1.536: 293.78}

# For parallel pipes
dic_candidateMergedDiam_costs={1.342: 235.4, 1.444: 263.66, 1.536: 293.78}

# %% tags=["nbval-skip"]
# Choose if a robust pipeline desin should be determined or the design should be optimized based on the
# given injection and withdrawal rates only
robust = True

# %% [markdown]
# ### Determine design for simple network structure

# %% tags=["nbval-skip"]
# Compute with 7 threads and simple network structure

dic_arc_optimalDiameters, dic_scen_PressLevels, dic_scen_MaxViolPress, dic_timeStep_PressLevels, \
    dic_timeStep_MaxViolPress, _ = robustPipelineSizing.determineDiscretePipelineDesign(
        robust=robust, injectionWithdrawalRates=rates,
        distances=lengths, dic_node_minPress=p_min_nodes, dic_node_maxPress=p_max_nodes,
        dic_diameter_costs=dic_diameter_costs, dic_candidateMergedDiam_costs=dic_candidateMergedDiam_costs,
        threads=7)

# %% tags=["nbval-skip"]
# Compute with 2 threads and simple network structure

dic_arc_optimalDiameters, dic_scen_PressLevels, dic_scen_MaxViolPress, dic_timeStep_PressLevels, \
    dic_timeStep_MaxViolPress, _ = robustPipelineSizing.determineDiscretePipelineDesign(
        robust=robust, injectionWithdrawalRates=rates,
        distances=lengths, dic_node_minPress=p_min_nodes, dic_node_maxPress=p_max_nodes,
        dic_diameter_costs=dic_diameter_costs, dic_candidateMergedDiam_costs=dic_candidateMergedDiam_costs,
        threads=2)

print("Finished Discrete Pipeline Optimization")

# %% [markdown]
# ### Determine design for more complex network structure

# %% tags=["nbval-skip"]
# Redfine minimum and maximum pressure levels to reduce robust scenario computation time
p_min_nodes = {'cluster_5': 50, 'cluster_3': 50, 'cluster_7': 50, 'cluster_1': 50, 'cluster_6': 50,
               'cluster_4': 50, 'cluster_0': 50, 'cluster_2': 50}

p_max_nodes = {'cluster_5': 100, 'cluster_3': 100, 'cluster_7': 100, 'cluster_1': 100, 'cluster_6': 100,
               'cluster_4': 100, 'cluster_0': 100, 'cluster_2': 100}

# %% tags=["nbval-skip"]
# Get more complex network structure
regColumn1 = 'loc1'
regColumn2 = 'loc2'

dic_node_minPress=p_min_nodes
dic_node_maxPress=p_max_nodes

maxPipeLength= 35 * 1e3
minPipeLength= 1 * 1e3

distances_new, dic_node_minPress_new, dic_node_maxPress_new, gdfNodes, gdfEdges = \
    robustPipelineSizing.getRefinedShapeFile(transFilePath, regColumn1, regColumn2, dic_node_minPress,
                                             dic_node_maxPress, minPipeLength, maxPipeLength)

fig, ax = fn.plotLocations(locFilePath, indexColumn='index')

gdfEdges.plot(ax=ax, color='grey')
gdfNodes.plot(ax=ax, color='red', markersize=2)
ax.axis('off')

plt.show()

# %% tags=["nbval-skip"]
# Determine optimal pipeline design with seven threads
dic_arc_optimalDiameters, dic_scen_PressLevels, dic_scen_MaxViolPress, dic_timeStep_PressLevels, \
           dic_timeStep_MaxViolPress, gdfEdges = robustPipelineSizing.determineDiscretePipelineDesign(
        robust=robust, injectionWithdrawalRates=rates,
        distances=distances_new, dic_node_minPress=dic_node_minPress_new, dic_node_maxPress=dic_node_maxPress_new,
        dic_diameter_costs=dic_diameter_costs, dic_candidateMergedDiam_costs=dic_candidateMergedDiam_costs,
        gdfEdges=gdfEdges, regColumn1='nodeIn', regColumn2='nodeOut', threads=7, solver='gurobi')

# %% [markdown]
# ### Plot scenario output

# %% tags=["nbval-skip"]
# Get regions shapefile as geopandas GeoDataFrame
gdf_regions = gpd.read_file(locFilePath)

# %% tags=["nbval-skip"]
# Visualize pipeline diameters
fig, ax =robustPipelineSizing.plotOptimizedNetwork(gdfEdges, gdf_regions=gdf_regions, figsize=(5,5),
    line_scaling=0.8)

# %% tags=["nbval-skip"]
# Get a minimum and maximum pressure scenario
dic_timeStep_PressLevels = pd.DataFrame.from_dict(dic_scen_PressLevels)

scen_min = dic_timeStep_PressLevels.mean().min()
scen_min = dic_timeStep_PressLevels.loc[:,dic_timeStep_PressLevels.mean() == scen_min].iloc[:,0]
scen_max = dic_timeStep_PressLevels.mean().max()
scen_max = dic_timeStep_PressLevels.loc[:,dic_timeStep_PressLevels.mean() == scen_max].iloc[:,0]

# %% tags=["nbval-skip"]
fig, ax =robustPipelineSizing.plotOptimizedNetwork(gdfEdges, gdf_regions=gdf_regions, figsize=(5,5),
    line_scaling=0.9, pressureLevels=scen_min)

# %% tags=["nbval-skip"]
fig, ax =robustPipelineSizing.plotOptimizedNetwork(gdfEdges, gdf_regions=gdf_regions, figsize=(5,5),
    line_scaling=0.9, pressureLevels=scen_max)

# %%


from project import Project
from buildadmin import BuildAdmin
from buildslave import BuildSlave
from builder import Builder
from repository import Repository, GitRepository, SVNRepository
from step import Step, RawShellCommandStep
from step import ShellCommandStep, CompileStep, ConfigureStep, TestStep
#from step import SVNCheckoutStep, GitCheckoutStep
from step import CheckoutStep
from notifier import Notifier, MailNotifier, WebNotifier
from scheduler import Scheduler, ChangeScheduler, CronScheduler

__all__ = [
    'Project',
    'BuildAdmin',
    'BuildSlave',
    'Builder',
    'Repository',
    'GitRepository',
    'SVNRepository',
    'Notifier',
    'MailNotifier',
    'WebNotifier',
    'Step',
    'RawShellCommandStep',
    'ShellCommandStep',
    'CompileStep',
    'ConfigureStep',
    'TestStep',
    #'SVNCheckoutStep',
    #'GitCheckoutStep',
    'CheckoutStep',
    'Scheduler',
    'ChangeScheduler',
    'CronScheduler',
]

# ===== register signal handlers =====
from django.db.models.signals import post_save, post_delete, m2m_changed

# signal handler
def update_project_status(sender, **kwargs):
    kwargs["instance"].project.update_status()

# For models with Project as ForeignKey
model_list = (
    BuildAdmin, BuildSlave, Builder,
    Repository, GitRepository, SVNRepository,
    Notifier, MailNotifier, WebNotifier,
    Scheduler, ChangeScheduler, CronScheduler,
)
for i, model in enumerate(model_list):
    for sig in (post_save,post_delete):
        sig.connect(update_project_status,
            sender=model,
            dispatch_uid='model-%d' % (i,) # unique uid to avoid duplicate entries
        )

# also trigger on m2m field changes
def update_on_m2m_change(sender, **kwargs):
    instance = kwargs["instance"]
    action   = kwargs["action"]
    if any(isinstance(instance, m) for m in (Builder, Scheduler)):
        if action in ("post_add", "post_remove"):
            instance.project.update_status()
m2m_changed.connect(update_on_m2m_change, dispatch_uid="m2m-post-save")

# Also update relevant project when first Step added or last Step deleted
def update_if_builder_empty(sender, **kwargs):
    obj = kwargs["instance"]
    if not isinstance(obj, Step): return
    builder = obj.builder
    if builder.project.has_builder and builder.step_set.count() == 0:
        builder.project.update_status()
post_delete.connect(update_if_builder_empty, dispatch_uid="model-step-del")

def update_if_builder_has_one_step(sender, **kwargs):
    """
    May miss an update if another step was added just before this method is
    called. We'll live with this for now.
    """
    obj = kwargs["instance"]
    if not isinstance(obj, Step): return
    if kwargs["created"]:
        obj.builder.project.update_status()
post_save.connect(update_if_builder_has_one_step, dispatch_uid="model-step-save")

# ===== auto-discover tests suites in ../tests =====
def suite():
    from bbotui.test_utils import locate_suites
    return locate_suites()


# -*- coding: windows-1251 -*-
import string
import math

f = open(r"res/1.txt", "r", encoding="windows-1251")
text = f.read()
text = text.lower()

def remove_chars_from_text(txt, chars):
    return ''.join([ch for ch in txt if ch not in chars])

spec_chars = string.punctuation + '«»–…1234567890—qwertyuiopasdfghjklzxcvbnm\n\t\xa0©'
text = remove_chars_from_text(text, spec_chars)
text = text.replace('¸', 'å')
text = text.replace('ú', 'ü')

text_tokens = dict()

def shannon(txt):
    for d in range(1, 10):
        summa = 0
        for i in range(len(txt) - d + 1):
            if text_tokens.get(txt[i:i + d]) is None:
                text_tokens[txt[i:i + d]] = 1
                summa += 1
            else:
                text_tokens[txt[i:i + d]] = text_tokens.get(txt[i:i + d]) + 1
                summa += 1
        if d == 1:
            max_h = math.log(len(text_tokens), 2)
            print("max_h = ", max_h)
        h = 0
        for value in text_tokens.values():
            h += (-1) * (value / summa) * math.log(value / summa, 2)
        h /= d
        print(f"h{d} = {h}")
        text_tokens.clear()

shannon(text)


import pygame

width = 800
lenght = 800
speed = 3

x, y = 0, 0
run = True

pygame.init()

pygame.time.Clock().tick(3)
pygame.key.set_repeat(1, 10)
win = pygame.display.set_mode((width, lenght))

road = pygame.image.load("road.png")
car = pygame.image.load("car.png")

road = pygame.transform.scale(road, (800, 800))

def Car(input_x, input_y):
    global x, y
    x += input_x * speed
    y += input_y * speed

while run:
    xst = yst = 0
    for event in pygame.event.get():
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_w: #z
                xst = 0
                yst = -1
            if event.key == pygame.K_s: #s
                xst = 0
                yst = 1
            if event.key == pygame.K_a: #q
                xst = -1
                yst = 0
            if event.key == pygame.K_d: #d
                xst = 1
                yst = 0
            if event.key == pygame.K_p:
                run = False
    Car(xst, yst)
    win.blit(road, (0, 0))
    win.blit(car, (190 + x, 190 + y))
    pygame.display.flip()
    print(x, y)


def get_IoC(strng):
    n=len(strng)
    IoC=0
    if n-1:
        IoC=(1/(float(n)*(n-1)))*(sum([strng.count(a)*(strng.count(a)-1) for a in set(strng)]))
    return(IoC)

def get_possible_key_ls(avg_IoC_arr):
    cpy=avg_IoC_arr.copy()
    avg_IoC_arr.sort(reverse=True)
    key_ls=[cpy.index(avg_IoC_arr[0])+2,cpy.index(avg_IoC_arr[1])+2]
    return(key_ls)

def get_key_len(strng,shift):
    avg_IoC_arr=[]
    for n in range(2,shift+1):
        IoC_sum=0.0
        avg_IoC=0.0
        for i in range(n):
            s=""
            for j in range(0,len(strng[i:]),n):
                s+=(strng[i:][j])
            IoC=get_IoC(s)
            IoC_sum+=IoC
        avg_IoC=IoC_sum/n
        avg_IoC_arr.append(avg_IoC)
    for k in avg_IoC_arr:
        print(str(avg_IoC_arr.index(k)+2)+" shift, IoC="+str(k))
    pos_key_ls=get_possible_key_ls(avg_IoC_arr)
    print("Best two possible key lengths:",pos_key_ls)

strng="QHDLXNQLYNGAIGWBCERJFEARNIBKXUSVGZXKYNPXXTKGAATZRQCRFYIDCCLYXHUQXEIXFAFGEAMMALYRGAYXQMTGACDJSYRTLEXUVRVIYFFEGXFKOYSPHGBBYTRESOXUNTXXAKLUAWYDINAAWCZWIFVMCROIUCEIFJYDJAYZJBEOTMUSGAGAYYQNIPTFPYMCBOYDYYSVGWDOJTBZLMFBYJXLQCUDRRIGMIUYWMQUUFRPCZQHTVJOUJSMNRVQQZEJYLACNHRFCPTFENZYEJCLYMBQUCGUMYQDBUAWLQTMOAXCZJBEABHQJYEAMQQDNIRLNTUINRMCYUJAQTZQMGOEXUDEONQPIDBXWNKNIEUNQMBQDUFGXLFXYBVKNTEZCBFJGJUTVHHMBWOZIFQNCTLMBQELYVGNTUHIAXNQUHSROYZJCEFUIACVOBFVAEGBBHGNEIMOHIYRIOZQ"
get_key_len(strng,14)

#Returns
#2 shift, IoC=0.0402126447693184
#3 shift, IoC=0.040925992938376836
#4 shift, IoC=0.040655987795575894
#5 shift, IoC=0.03879598662207358
#6 shift, IoC=0.042133363712311085
#7 shift, IoC=0.06341991341991342
#8 shift, IoC=0.039876091089793446
#9 shift, IoC=0.039084967320261434
#10 shift, IoC=0.03768115942028986
#11 shift, IoC=0.04353816914792524
#12 shift, IoC=0.04320616162721425
#13 shift, IoC=0.039668174962292614
#14 shift, IoC=0.060213308197179176
#Best two possible key lengths: [7, 14]


import pandas as pd

# 先创建一个数据框（包含缺失值）
df = pd.DataFrame({'auth': ['spring', 'summer', 'fall', 'spring'],
                   'sply': ['a', 'c', 'a', 'b'],
                   'name': ['zhangsan', 'lisi', 'xiaohua', 'xiaomei']})

# categorical_name = ['auth', 'sply', 'name']
categorical_name = ['name']

print(df)

# 定义一个循环函数，处理分类型特征，进行标签编码
def categorical_preprocessing(dataset, categorical_feature):
    '''
    param:
        dataset:DataFrame,输入的数据集
        categorical_feature:list,分类特征列名
    '''
    for feature in categorical_feature:
        set_feature = set(dataset[feature])  # 将特征映射到集合中
        dic_feature = {}
        for i, feat in enumerate(set_feature):
            dic_feature[feat] = i
            # print("feat:", feat, "index:", i)
        # dataset[feature] = dataset[feature].map(dic_feature)
        dataset[feature] = dataset.set_index([feature]).index.map(dic_feature.get)
    # dataset = pd.get_dummies(dataset, columns=categorical_feature)
    return dataset

# 处理分类特征编码
dataset = categorical_preprocessing(df, categorical_name)

print(dataset)
print("====")
print(dataset["name"].unique().tolist())


from django.shortcuts import render
# from django.urls import resolve
from mainapp.models import ProductCategory
from mainapp.models import Product
from basket.models import BasketSlot
from basket.views import basket as basket_view
# from django.shortcuts import get_object_or_404

# Create your views here.

def main(request):
    context = {
        'title': 'главная страница'
    }
    return render(request, 'mainapp/main.html', context)

def products(request, category_pk=0):
    basket = {'basket': None, 'total_cost': 0}

    product_categories = ProductCategory.objects.all()

    if category_pk == 0:
        product_list = Product.objects.all()
    else:
        product_list = Product.objects.filter(category=category_pk)

    if request.user.is_authenticated:
        # print(request.user.basket.all())
        basket = basket_view(request)

    context = {
        'title': 'наши продукты',
        'prod_cats': product_categories,
        'basket': basket['basket'],
        'total_cost': basket['total_cost'],
        'product_list': product_list,
    }
    return render(request, 'mainapp/products.html', context)

def contacts(request):
    context = {
        'title': 'наши контакты'
    }
    return render(request, 'mainapp/contacts.html', context)


"""GOST 28147-89 (aka Magma) ecnryption algorithm."""

import secrets
from Crypto.Util.strxor import strxor
import pygost.gost3412 as gost3412

import utils

class Magma():
    """Class contains Magma algorithm."""

    def __init__(self):
        # Size of key in bytes
        self.key_size = gost3412.KEYSIZE
        # Size of initialization vector in bytes
        self.iv_size = gost3412.GOST3412Magma.blocksize
        # Size of key data in bytes
        self.key_data_size = self.key_size + self.iv_size
        # Size of the data block read to encrypt
        self.data_block_size = gost3412.GOST3412Magma.blocksize

    def keygen(self):
        """Generates a random key."""

        return secrets.token_bytes(self.key_data_size)

    def encrypt(self, ifstream, ofstream, key_data):
        """Encrypts data using Magma algorithm in CBC mode.

        :param ifstream: binary input stream
        :type ifstream: BufferedReader

        :param ofstream: binary output stream
        :type ofstream: BufferedWriter

        :param key_data: concatenation of key and initializaton vector for the algorithm
        :type key_data: bytes
        """

        key = key_data[:self.key_size]
        init_vector = key_data[self.key_size:]

        magma = gost3412.GOST3412Magma(key)
        tmp_data = init_vector

        data_in = ifstream.read(self.data_block_size)
        while data_in != b'':
            if len(data_in) < self.data_block_size:
                data_in = utils.pad(data_in, self.data_block_size)
            data_in = strxor(data_in, tmp_data)
            data_out = magma.encrypt(data_in)
            tmp_data = data_out
            ofstream.write(data_out)
            data_in = ifstream.read(self.data_block_size)

    def decrypt(self, ifstream, ofstream, key_data, data_size):
        """Decrypts data using Magma algorithm in CBC mode.

        :param ifstream: binary input stream
        :type ifstream: BufferedReader

        :param ofstream: binary output stream
        :type ofstream: BufferedWriter

        :param key_data: concatenation of key and initializaton vector for the algorithm
        :type key_data: bytes

        :param data_size: size of the payload
        :type data_size: int
        """

        key = key_data[:self.key_size]
        init_vector = key_data[self.key_size:]

        magma = gost3412.GOST3412Magma(key)
        tmp_data = init_vector

        data_in = ifstream.read(self.data_block_size)
        while data_in != b'':
            data_out = magma.decrypt(data_in)
            data_out = strxor(data_out, tmp_data)
            tmp_data = data_in
            if data_size < self.data_block_size:
                data_out = utils.unpad(data_out, data_size)
            ofstream.write(data_out)
            data_in = ifstream.read(self.data_block_size)
            data_size -= len(data_in)


"""Model implementation for Windows."""

import os
import json
import logging
from pathlib import Path
from random import randint
from collections import Counter
from typing import Any, NamedTuple

from PIL import Image

from murphy.model.utils import absolute_coordinates
from murphy.model import scrapers, Interpreter, State, Window
from murphy.model import Action, Button, TextBox, Link, ComboBox
from murphy.model.interpreters.windows_image import compare_images

Tolerance = NamedTuple('Tolerance', (('image', float),  # image comparison
                                     ('load', tuple)))  # device load comparison
DEFAULT_TOLERANCE = Tolerance(1.0, (0.4, 0.2, 0.1))

class WindowsInterpreter(Interpreter):
    """The WindowsState Interpreter allows to construct the State
    of a Windows-based GUI application.

    A tolerance attribute is provided for tuning State comparison logic.

    Check the WindowsState documentation for further information.

    """
    tolerance = DEFAULT_TOLERANCE  # Type: Tolerance
    """Maximum tolerance for window image comparison and device load."""

    _screen_size = None            # Type: tuple
    """Screen resolution in pixels."""

    def __init__(self, feedback: Any, control: Any,
                 scraper: scrapers.WindowScraper):
        self.control = control
        self.scraper = scraper
        self.feedback = feedback
        self.logger = logging.getLogger("%s.%s" % (self.__module__,
                                                   self.__class__.__name__))

    def interpret_state(self) -> 'WinState':
        self._move_cursor_away()

        feedback = self._raw_feedback()
        image = Image.open(feedback.image).crop(feedback.scraped.coordinates)
        window = WindowsWindow(feedback.scraped, image)
        actions = tuple(ACTIONS[o.type](self.control, o, window)
                        for o in feedback.scraped.objects if o.type in ACTIONS)

        return WindowsState(
            self.control, window, actions, feedback, self.tolerance)

    def load_state(self, path: Path) -> 'WinState':
        feedback = self._raw_feedback(path)
        image = Image.open(feedback.image)
        window = WindowsWindow(feedback.scraped, image)
        actions = tuple(ACTIONS[o.type](self.control, o, window)
                        for o in feedback.scraped.objects if o.type in ACTIONS)

        return WindowsState(
            self.control, window, actions, feedback, self.tolerance)

    def _raw_feedback(self, path: Path = None) -> 'RawFeedback':
        """Retrieve raw information from the Feedback class
        or from a previously dumped state.

        Order of calls matters.

        """
        saved = None

        if path is not None:
            with path.open() as state_file:
                state = json.load(state_file)

            screenshot = state['window']
            saved = state['state']
            device_load = state['load']
            scraped = load_scraped_window(state['scraped'])
        else:
            load = self.feedback.load
            device_load = load.cpu, load.disk, load.network
            scraped = self.scraper.scrape_current_window()
            screenshot = self.feedback.screen.screenshot()

        return RawFeedback(device_load, screenshot, scraped, saved)

    def _move_cursor_away(self):
        """Move the mouse cursor away from the screen
        to avoid it interfering with the screenshot.

        """
        if self._screen_size is None:
            screeshot_path = self.feedback.screen.screenshot()
            self._screen_size = Image.open(screeshot_path).size

        self.control.mouse.move(self._screen_size[0] - 1,
                                self._screen_size[1] - 1)

class WindowsState(State):
    """Class representing the State of a Windows-based GUI application.

    Comparison against other states is done via comparing the window title,
    the list of available actions and computing the root mean squared distance
    between the images of the two windows.

    A tolerance attribute is provided for tuning the State comparison logic.
    If the Root Mean Squared Distance of the State images
    exceeds the image Tolerance, the two images are considered different.
    If the cpu, disk or network loads exceed the load Tolerance, the device
    is considered busy at the moment the State was analysed.

    """
    _saved_state = None  # Type: Any

    def __init__(self, control: Any, window: Window, actions: list,
                 feedback: 'RawFeedback', tolerance: Tolerance):
        self._control = control

        self.window = window
        self.actions = actions
        self.tolerance = tolerance
        self.raw_feedback = feedback
        self.logger = logging.getLogger("%s.%s" % (self.__module__,
                                                   self.__class__.__name__))

    def __str__(self) -> str:
        return self.window.title

    def __repr__(self) -> str:
        return repr(self.raw_feedback.scraped)

    def __eq__(self, state: State) -> bool:
        """Compare the current state with the given one.

        If the window titles, the actions or the window images differ,
        then the States are considered different.

        """
        if self.window.title != state.window.title:
            return False

        if Counter(self.actions) != Counter(state.actions):
            return False

        return compare_images(
            self.window.image, state.window.image,
            self.tolerance.image, self.actions)

    @property
    def busy(self) -> bool:
        """The device is considered busy when the cpu, disk or network loads
        exceed the load Tolerance.

        """
        self.logger.info(
            "Load: CPU %f - Disk %f - Network %f", *self.raw_feedback.load)

        for load, tolerance in zip(self.raw_feedback.load, self.tolerance.load):
            if load > tolerance:
                return True

        return False

    def save(self):
        if self._saved_state is None:
            self._saved_state = self._control.state.save()
            self.logger.info("State %s saved", self.window.title)
        else:
            raise RuntimeError("State already saved")

    def restore(self):
        if self._saved_state is not None:
            self._control.state.restore(self._saved_state)
            self.logger.info("State %s restored", self.window.title)
        else:
            raise RuntimeError("State was not saved")

    def discard(self):
        if self._saved_state is not None:
            self._control.state.discard(self._saved_state)
            self.logger.info("State %s discarded", self.window.title)
        else:
            raise RuntimeError("State was not saved")

    def dump(self, path: Path):
        image_path = path.joinpath('window.png')
        state_path = path.joinpath('state.json')

        path.mkdir(parents=True, exist_ok=True)
        self.window.image.save(image_path)

        state = {'window': str(image_path),
                 'state': self._saved_state,
                 'load': self.raw_feedback.load,
                 'scraped': self.raw_feedback.scraped._asdict()}

        with state_path.open('w') as state_file:
            json.dump(state, state_file)

        self.logger.info("State %s dumped at %s", self.window.title, state_path)

class WindowsWindow(Window):
    def __init__(self, scraped_window: scrapers.Window, image: Image):
        self._image = image

        self.title = scraped_window.text
        self.coordinates = scraped_window.coordinates
        self.text = os.linesep.join(o.text for o in scraped_window.objects
                                    if o.type == scrapers.ObjectType.STATIC)

    @property
    def image(self) -> Image:
        return self._image.copy()

class WindowsAction(Action):
    focused = False
    """Actions under keyboard focus."""

    def __init__(self, control: Any, scraped: scrapers.Object, window: Window):
        self._window = window
        self._control = control

        self.text = scraped.text
        self.coordinates = scraped.coordinates
        self.focused = scraped.properties.get('focused', False)

    def __eq__(self, action: Action) -> bool:
        return self.text == action.text

    def __hash__(self):
        return hash(self.__class__.__name__) + hash(self.text)

    @property
    def image(self) -> Image:
        return self._window.image.crop(self.coordinates)

    def _cursor_coordinates(self) -> tuple:
        """Select a random pair of coordinates within the action area.

        The mouse can use the coordinates for performing its actions.

        """
        coordinates = absolute_coordinates(
            self.coordinates, self._window.coordinates)

        return (randint(coordinates.left + 1, coordinates.right - 1),
                randint(coordinates.top + 1, coordinates.bottom - 1))

    def _move_cursor_away(self):
        """Move the mouse cursor away from the Window
        to avoid it interfering with the screenshot.

        """
        self._control.mouse.move(
            self._window.coordinates.right, self.coordinates.bottom)

class WindowsButton(Button, WindowsAction):
    toggled = False
    """Checkboxes and radio buttons will be toggled if ticked."""

    def __init__(self, control: Any, scraped: scrapers.Object, window: Window):
        super().__init__(control, scraped, window)
        self.toggled = scraped.properties.get('toggled', False)

    def __eq__(self, action: Action) -> bool:
        return super().__eq__(action) and self.toggled == action.toggled

    def __hash__(self):
        return super().__hash__() + int(self.toggled)

    def perform(self):
        """Single left-button click with the mouse."""
        self._control.mouse.move(*self._cursor_coordinates())
        self._control.mouse.click()
        self._move_cursor_away()

class WindowsTextBox(TextBox, WindowsAction):
    def perform(self, text: str):
        self._control.mouse.move(*self._cursor_coordinates())
        self._control.mouse.click()
        self._move_cursor_away()

        self._control.keyboard.type(text)

class WindowsLink(Link, WindowsAction):
    def perform(self):
        """Single left-button click with the mouse."""
        self._control.mouse.move(*self._cursor_coordinates())
        self._control.mouse.click()
        self._move_cursor_away()

class WindowsComboBox(ComboBox, WindowsAction):
    """Combo Boxes are also referred as drop-down lists.

    The drop-down elements are listed within the items attribute.

    """
    def __init__(self, control, scraped: scrapers.Object, window: Window):
        super().__init__(control, scraped, window)

        self.items = tuple(scraped.properties.get('items', ()))

    def perform(self, positions: int):
        """Select the drop-down item in the list of items.

        The position parameter chooses how many positions to move the selector.
        Positive numbers move the selector down, negative ones up.

        """
        if abs(positions) > len(self.items):
            raise ValueError("positions out of range")

        self._control.mouse.move(*self._cursor_coordinates())
        self._control.mouse.click()
        self._move_cursor_away()

        key = 'down' if positions > 0 else 'up'
        for _ in range(abs(positions)):
            self._control.keyboard.press(key)

        self._control.keyboard.press('return')

def load_scraped_window(state: dict) -> scrapers.Window:
    """Reconstruct a scraped window from a State dump."""
    objects = tuple(scrapers.Object(*o) for o in state['objects'])

    return scrapers.Window(state['text'], objects, state['coordinates'],
                           state['scraper'], state['raw'])

ACTIONS = {scrapers.ObjectType.BUTTON: WindowsButton,
           scrapers.ObjectType.TEXTBOX: WindowsTextBox,
           scrapers.ObjectType.LINK: WindowsLink,
           scrapers.ObjectType.COMBOBOX: WindowsComboBox}
RawFeedback = NamedTuple('RawFeedback', (('load', tuple),
                                         ('image', Path),
                                         ('scraped', scrapers.Window),
                                         ('saved_state', Any)))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import re

from rest_framework import serializers

class Field(serializers.Serializer):
    """
    Abstract base class for a compound field.
    =========================================

    Subclasses must implement a validate(self, attrs) method.

    They must also have a "_field_name" attribute that is unique. This is used
    to identify the class in the database and to the frontend. And a
    "_field_display_name" attribute that is displayed to users.

    Also an "options_doc" dictionary attribute must be supplied documenting any
    options that can be stored in the database.
    """
    def __init__(self, *args, **kwargs):
        """
        Optionally provide an "options" dictionary that can be used by for
        more granular validation.
        """
        self.options = kwargs.pop('options', {})
        super(Field, self).__init__(*args, **kwargs)

    def validate(self, attrs):
        raise NotImplementedError("Field subclasses must provide their own "
                                  "validation logic.")

    @property
    def data(self):
        obj = self.object
        obj['_field'] = {"name": self._field_name, "options": self.options}
        return self.object

    def metadata(self):
        metadata = super(Field, self).metadata()
        return {
            'name': self._field_name,
            'documentation': self.__doc__.strip(),
            'complex': True,
            'fields': metadata,
            'options': self.options_doc,
        }

class SimpleFieldMeta(serializers.SerializerMetaclass):
    """
    Metaclass for SimpleField.
    ==========================

    Ensures that a SimpleField instance has only one serializers.Field
    attribute called "value".
    """
    def __new__(meta, name, bases, attrs):
        if name != 'SimpleField':
            if 'value' not in attrs:
                for base in bases:
                    valueField = getattr(base, 'base_fields', {}).get('value')
                    if valueField:
                        attrs['value'] = valueField
                        break

            if ('value' not in attrs
                    or not isinstance(attrs['value'], serializers.Field)):
                raise NotImplementedError(
                    'Class {} must have a "value" field.'.format(name))

            for key, attr in attrs.iteritems():
                if key == 'value': continue
                if isinstance(attr, serializers.Field):
                    raise TypeError(
                        ('Class {} must only have a "value" field. '
                         '"{}" field found.').format(name, key)
                    )
        return super(SimpleFieldMeta, meta).__new__(meta, name, bases, attrs)

class SimpleField(Field):
    """
    Abstract base class for a scalar field.
    =======================================

    All instances must have one and only one serializers.Field attribute and it
    must be called "value".
    """
    __metaclass__ = SimpleFieldMeta

    def validate(self, attrs):
        return attrs

    def metadata(self):
        metadata = super(SimpleField, self).metadata()
        metadata["complex"] = False
        metadata['fields'] = metadata['fields']['value']
        return metadata

class TextField(SimpleField):
    """
    Plain text (multiline)
    """
    _field_name = "textarea"
    _field_display_name = "Text (TextArea)"

    options_doc ={
        'max_length': {
            'description': "Maximum Length",
            'type': 'integer',
            'required': False
        },
        'min_length': {
            'description': "Minimum Length",
            'type': 'integer',
            'required': False
        },
    }

    value = serializers.CharField()

    def validate_value(self, attrs, source):
        value = attrs[source]
        max_length = self.options.get('max_length', None)
        min_length = self.options.get('min_length', 0)
        if max_length is not None and len(value) > max_length:
            raise serializers.ValidationError(
                'over maximum length {}'.format(max_length)
            )
        if len(value) < min_length:
            raise serializers.ValidationError(
                'under minimum length {}'.format(min_length)
            )
        return attrs

class CharField(TextField):
    """
    Plain text
    """
    _field_name = "text"
    _field_display_name = "Text (Single Line)"

class IntegerField(SimpleField):
    """
    Integer
    """
    _field_name = "int"
    _field_display_name = "Integer"
    options_doc ={
        'max': {
            'description': "Maximum Value",
            'type': 'integer',
            'required': False
        },
        'min': {
            'description': "Minimum Value",
            'type': 'integer',
            'required': False
        },
    }

    value = serializers.IntegerField()

    def validate_value(self, attrs, source):
        value = attrs[source]
        max_val = self.options.get('max', None)
        min_val = self.options.get('min', None)
        if max_val is not None and value > max_val:
            raise serializers.ValidationError(
                "over maximum value {}".format(max_val)
            )
        if min_val is not None and value < min_val:
            raise serializers.ValidationError(
                "under minimum value {}".format(min_val)
            )
        return attrs

class FloatField(SimpleField):
    """
    Floating Point Number
    """
    _field_name = "float"
    _field_display_name = "Float"
    options_doc ={
        'max': {
            'description': "Maximum Value",
            'type': 'float',
            'required': False
        },
        'min': {
            'description': "Minimum Value",
            'type': 'float',
            'required': False
        },
    }
    value = serializers.FloatField()

    def validate_value(self, attrs, source):
        value = attrs[source]
        max_val = self.options.get('max', None)
        min_val = self.options.get('min', None)
        if max_val is not None and value > max_val:
            raise serializers.ValidationError(
                "over maximum value {}".format(max_val)
            )
        if min_val is not None and value < min_val:
            raise serializers.ValidationError(
                "under minimum value {}".format(min_val)
            )
        return attrs

class PhoneNumberField(TextField):
    """
    Phone number with only digits, spaces, and dashes.
    """
    _field_name = "phone"
    _field_display_name = "Phone Number"
    options_doc ={}
    value = serializers.RegexField('^[\d -]+$')

class USAddressField(Field):
    """
    United States of America street address.
    ========================================

    Has 2 street fields, and a city field. A state field is limited to choices
    of valid US states and territories. A zip code validates to 5-digit or
    9-digit numeric zip codes.
    """
    _field_name = "address-us"
    _field_display_name = "Address (United States)"
    options_doc ={}

    STATE_CHOICES = (
        ('AK', 'Alaska'),
        ('AL', 'Alabama'),
        ('AR', 'Arkansas'),
        ('AS', 'American Samoa'),
        ('AZ', 'Arizona'),
        ('CA', 'California'),
        ('CO', 'Colorado'),
        ('CT', 'Connecticut'),
        ('DC', 'District of Columbia'),
        ('DE', 'Delaware'),
        ('FL', 'Florida'),
        ('GA', 'Georgia'),
        ('GU', 'Guam'),
        ('HI', 'Hawaii'),
        ('IA', 'Iowa'),
        ('ID', 'Idaho'),
        ('IL', 'Illinois'),
        ('IN', 'Indiana'),
        ('KS', 'Kansas'),
        ('KY', 'Kentucky'),
        ('LA', 'Louisiana'),
        ('MA', 'Massachusetts'),
        ('MD', 'Maryland'),
        ('ME', 'Maine'),
        ('MI', 'Michigan'),
        ('MN', 'Minnesota'),
        ('MO', 'Missouri'),
        ('MP', 'Northern Mariana Islands'),
        ('MS', 'Mississippi'),
        ('MT', 'Montana'),
        ('NA', 'National'),
        ('NC', 'North Carolina'),
        ('ND', 'North Dakota'),
        ('NE', 'Nebraska'),
        ('NH', 'New Hampshire'),
        ('NJ', 'New Jersey'),
        ('NM', 'New Mexico'),
        ('NV', 'Nevada'),
        ('NY', 'New York'),
        ('OH', 'Ohio'),
        ('OK', 'Oklahoma'),
        ('OR', 'Oregon'),
        ('PA', 'Pennsylvania'),
        ('PR', 'Puerto Rico'),
        ('RI', 'Rhode Island'),
        ('SC', 'South Carolina'),
        ('SD', 'South Dakota'),
        ('TN', 'Tennessee'),
        ('TX', 'Texas'),
        ('UT', 'Utah'),
        ('VA', 'Virginia'),
        ('VI', 'Virgin Islands'),
        ('VT', 'Vermont'),
        ('WA', 'Washington'),
        ('WI', 'Wisconsin'),
        ('WV', 'West Virginia'),
        ('WY', 'Wyoming')
    )

    street1 = serializers.CharField()
    street2 = serializers.CharField(required=False)
    city = serializers.CharField()
    state = serializers.ChoiceField(choices=STATE_CHOICES)
    zip = serializers.RegexField(r'\d{5}(-\d{4})?')

    def validate(self, attrs):
        return attrs

class DisplayOnlyField(SimpleField):
    """
    Display a Paragraph of Text
    ===========================

    This field does not take input from the user, but provides the option for
    a form publisher to inject some text for example to give instructions to the
    form user anywhere into the form.
    """
    _field_name = "display-only"
    _field_display_name = "Display Only"
    options_doc ={
        "text": {
            "description": "Text to Display",
            "type": "string",
            "required": True,
        }
    }
    value = serializers.WritableField(required=False)

    def validate_value(self, attrs, source):
        attrs[source] = None
        return attrs

ALL_FIELDS = [TextField,
              CharField,
              IntegerField,
              FloatField,
              PhoneNumberField,
              USAddressField,
              DisplayOnlyField]


from flask import Flask, render_template, request, session
import pickle
import pandas as pd
import lang_maker

app = Flask(__name__)
app.secret_key = 'hello'

score_tabl = pd.read_csv('/static/words/score_tabl_simplified.csv', index_col='Unnamed: 0', keep_default_na=False) # prevent string 'nan' interpreted as NaN
score_tabl2_3 = pd.read_csv('/static/words/score_tabl2_3.csv', index_col='Unnamed: 0', keep_default_na=False) # prevent string 'nan' interpreted as NaN
lang_digs = pd.read_csv('/static/words/lang_digs.csv', index_col='Unnamed: 0')
leng_dists = pd.read_csv('/static/words/leng_dists.csv', index_col='Unnamed: 0')
with open('/static/words/score_tabl_dict_simplified.pkl', 'rb') as f:
    score_tabl_dict = pickle.load(f)
with open('/static/words/score_tabl_dict2_3.pkl', 'rb') as f:
    score_tabl_dict2_3 = pickle.load(f)
with open('/static/words/char_dist_init.pkl', 'rb') as f:
    char_dist_init = pickle.load(f)
with open('/static/words/letter_distributions.pkl', 'rb') as f:
    letter_distributions = pickle.load(f)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        lang_input = {'aus': int(request.form['aus-pr'][:-1]) / 100,
                        'ban': int(request.form['ban-pr'][:-1]) / 100,
                        'fin': int(request.form['fin-pr'][:-1]) / 100,
                        'rom': int(request.form['rom-pr'][:-1]) / 100,
                        'tur': int(request.form['tur-pr'][:-1]) / 100}
        influence = list(lang_input.values())
        search_time = request.form['word-time']

        # Make Sentence
        sentence = lang_maker.generate_sentence(lang_digs, leng_dists, influence, search_time)
        session['sentence'] = sentence
        return render_template('index.html', sentence=sentence)
    else:
        if len(session) > 0:
            sentence = session['sentence']
            return render_template('index.html', sentence=sentence)
        else:
            sentence = "Result here."
            return render_template('index.html', sentence=sentence)

if __name__ == '__main__':
    app.run(debug=True)


for loopCounter in range(5):
    print(loopCounter)

userInput = input("What would you like to type in? ")

def userPrint():
    for loopCounter in range(2):
        print (userInput)

userPrint()

def userNumberFunction():
    userNumber = int(input("Enter a number between 1 and 10? "))
    if (userNumber < 11):
        print(userNumber)

userNumberFunction()

def whileLoopCounter():
    userNumberInput = int(input("What number would you like to input? "))
    while (userNumberInput != 0):
        print(userNumberInput)
        break

whileLoopCounter()

def whileLoopPrint():
    userNumberInput2 = (input("Enter the number 1, 2, 3, or 0: "))
    while (userNumberInput2 != "0"):
        if (userNumberInput2 == "1"):
            print("1")
            break
        elif (userNumberInput2 == "2"):
            print("2")
            break
        elif (userNumberInput2 == "3"):
            print("3")
            break
        elif (userNumberInput2 == "0"):
            break
        else:
            print("ERROR")
            break

whileLoopPrint()

userNumberInput3 = input("Enter a number between 1 and 3 or q to quit: ").lower()

def one():
    while (userNumberInput3 != "q"):
        if (userNumberInput3 == "1"):
            print("1")
            break

def two():
    while (userNumberInput3 != "q"):
        if (userNumberInput3 == "2"):
            print("2")
            break

def three():
    while (userNumberInput3 != "q"):
        if (userNumberInput3 == "3"):
            print("3")
            break

def quit():
        if(userNumberInput3 != "1" and userNumberInput3 != "2"
                and userNumberInput3 != "3" and userNumberInput3 != "q" ):
            return ("Error")

def allTogether():
    if (userNumberInput3 == "1"):
        one()
    elif(userNumberInput3 == "2"):
        two()
    elif(userNumberInput3 == "3"):
        three()
    elif(userNumberInput3 == "q"):
        print("Project terminated")
    else:
        print(quit())

allTogether()


from django.conf.urls import url

from . import views

app_name = 'gatekeeper'
urlpatterns = [
	url(r'^$', views.index, name='index'),
	url(r'^(?P<sugestion_id>[0-9]+)/$', views.sugestions, name='sugestion'),
	url(r'^add_sugestion/$', views.add_sugestion, name='add_sugestion'),
	url(r'^sign_in/$', views.sign_in, name='sign_in'),
]


#!/usr/bin/env python
#coding: utf-8

import datetime
import models

class Message(object):
    def __init__(self):
        today = datetime.datetime.today()
        self._wday = today.isoweekday()
        self._mday = today.day

        self._message_model = models.message()

    @property
    def current_person(self):
        return self._message_model.members[0]

    @property
    def next_person(self):
        return self._message_model.members[1]

    @property
    def daily_message(self):
        return self._message_model.daily_messages[self._wday]

    @property
    def template(self):
        is_week_end = self._wday == 6
        is_month_end = self._mday == 26

        if is_week_end and is_month_end:
            s = self._message_model.templates[0]
        elif is_month_end:
            s = self._message_model.templates[1]
        elif is_week_end:
            s = self._message_model.templates[2]
        else:
            s = self._message_model.templates[3]
        return s

    def __unicode__(self):
        current_person = self.current_person
        next_person = self.next_person
        daily_message = self.daily_message
        return self.template % locals()
#        return self.template


import RPi.GPIO as GPIO
import time
import numpy as np

def main():
    GPIO.setmode(GPIO.BCM)
    GPIO.setwarnings(False)

    # Variablen
    adcChannel = 0

    # Pins
    SPI_SCLK = 18
    SPI_MOSI = 24
    SPI_MISO = 23
    SPI_CS = 25

    GPIO.setup(SPI_SCLK, GPIO.OUT)
    GPIO.setup(SPI_MOSI, GPIO.OUT)
    GPIO.setup(SPI_MISO, GPIO.IN)
    GPIO.setup(SPI_CS, GPIO.OUT)

    # Start Messung
    messdauer = 10      # Zeit in Sekunden
    startTime = time.time()

    timeList = []
    amplitudeList = []
    while time.time() - startTime < messdauer:
        timeList.append( time.time() - startTime )
        adcVal = readADC(adcChannel, SPI_SCLK, SPI_MOSI, SPI_MISO, SPI_CS)
        amplitudeList.append( adcToPercent(adcVal) )
    # Ende Messung

    # Speicherung Amplitudendaten
    amplitudeData = np.asarray( [timeList, amplitudeList] ).T
    np.savetxt('amplitudeData.dat', amplitudeData, header='Time (s)\tAmplitude (\%)')

    # Erzeugung und Speicherung der Freuqenzdaten
    freq, freqAmplitude = calculateFFT(timeList, amplitudeList)
    frequencyData = np.asarray( [freq, abs(freqAmplitude), np.real(freqAmplitude), np.imag(freqAmplitude)] ).T
    np.savetxt('frequencyData.dat', frequencyData, header='Frequency (Hz)\tAmplitude (a.u.)\tRealteil\tImaginaerteil')

def readADC(adcChannel, SCLKPin, MOSIPin, MISOPin, CSPin):
    # Selektiere den ADC
    GPIO.output(CSPin, True)
    GPIO.output(CSPin, False)
    GPIO.output(SCLKPin, False)

    cmd = adcChannel
    cmd |= 0b11000
    cmdLength = 5

    # Senden des Kommandos
    for i in range(cmdLength):
        if cmd & (1 << (cmdLength - i - 1)):
            GPIO.output(MOSIPin, True)
        else:
            GPIO.output(MOSIPin, False)

        # Ein Clockpuls
        GPIO.output(SCLKPin, True)
        GPIO.output(SCLKPin, False)

	# Empfangen von Daten
    response = 0
    responseLength = 12

    # Empfange Nullbit
    GPIO.output(SCLKPin, True)
    GPIO.output(SCLKPin, False)

    # Empfange Daten
    for i in range(responseLength):
        GPIO.output(SCLKPin, True)
        GPIO.output(SCLKPin, False)

        if GPIO.input(MISOPin):
            response |= (1 << (responseLength - i - 1))

    return response

def adcToPercent(adcVal):
    adcResolution = float(2**12 - 1)
    voltVal = adcVal/adcResolution * 3.3

    # Spannung relativ zum halben Maximum
    voltVal = voltVal - 3.3 / 2

    # Normierung
    voltVal /= (3.3 / 2)

    return voltVal

def calculateFFT(time, amplitude):
    # Mittlere Zeitdifferenz zwischen zwei Datenpunkten
    tMean = np.mean(np.diff(time))

    # Erzeuge neue Zeiten
    timeNew = np.arange(min(time), max(time), tMean)

    # Interpolation
    amplitudeNew = np.interp(timeNew, time, amplitude)

    # Calculate FFT
    freq = np.fft.rfftfreq(timeNew.size, tMean)
    freqAmplitude = np.fft.rfft(amplitudeNew)

    return freq, freqAmplitude

if __name__ == '__main__':
    main()


import numpy as np
import theano
from scipy import stats

from neupy import layers, surgery
from neupy.layers.normalization import find_opposite_axes

from base import BaseTestCase

class BatchNormTestCase(BaseTestCase):
    def test_find_pposite_axis_invalid_cases(self):
        with self.assertRaises(ValueError):
            find_opposite_axes(axes=[5], ndim=1)

        with self.assertRaises(ValueError):
            find_opposite_axes(axes=[0, 1], ndim=1)

    def test_find_pposite_axis_valid_cases(self):
        testcases = (
            dict(input_kwargs={'axes': [0, 1], 'ndim': 4},
                 expected_output=[2, 3]),
            dict(input_kwargs={'axes': [], 'ndim': 4},
                 expected_output=[0, 1, 2, 3]),
            dict(input_kwargs={'axes': [0, 1, 2], 'ndim': 3},
                 expected_output=[]),
        )

        for testcase in testcases:
            actual_output = find_opposite_axes(**testcase['input_kwargs'])
            self.assertEqual(actual_output, testcase['expected_output'],
                             msg="Kwargs: ".format(testcase['input_kwargs']))

    def test_simple_batch_norm(self):
        connection = layers.Input(10) > layers.BatchNorm()
        connection.initialize()

        input_value = theano.shared(value=np.random.random((30, 10)))
        output_value = connection.output(input_value).eval()

        self.assertTrue(stats.mstats.normaltest(output_value))
        self.assertAlmostEqual(output_value.mean(), 0, places=3)
        self.assertAlmostEqual(output_value.std(), 1, places=3)

    def test_batch_norm_gamma_beta_params(self):
        default_beta = -3.14
        default_gamma = 4.3
        connection = layers.Input(10) > layers.BatchNorm(gamma=default_gamma,
                                                         beta=default_beta)
        connection.initialize()

        input_value = theano.shared(value=np.random.random((30, 10)))
        output_value = connection.output(input_value).eval()

        self.assertAlmostEqual(output_value.mean(), default_beta, places=3)
        self.assertAlmostEqual(output_value.std(), default_gamma, places=3)

    def test_batch_norm_between_layers(self):
        connection = surgery.sew_together([
            layers.Input(10),
            layers.Relu(40),
            layers.BatchNorm(),
            layers.Relu(1),
        ])
        connection.initialize()

        input_value = np.random.random((30, 10))
        outpu_value = connection.output(input_value).eval()

        self.assertEqual(outpu_value.shape, (30, 1))

    def test_batch_norm_exceptions(self):
        with self.assertRaises(ValueError):
            connection = layers.Input(10) > layers.BatchNorm(axes=2)
            connection.initialize()

        with self.assertRaises(ValueError):
            connection = layers.Relu() > layers.BatchNorm()
            connection.initialize()

    def test_batch_norm_in_non_training_state(self):
        batch_norm = layers.BatchNorm()
        conneciton = layers.Input(10) > batch_norm
        batch_norm.initialize()

        input_value = theano.shared(value=np.random.random((30, 10)))

        self.assertEqual(len(batch_norm.updates), 0)
        batch_norm.output(input_value)
        self.assertEqual(len(batch_norm.updates), 2)

        with batch_norm.disable_training_state():
            # Without training your running mean and std suppose to be
            # equal to 0 and 1 respectavely.
            output_value = batch_norm.output(input_value).eval()
            np.testing.assert_array_almost_equal(
                input_value.get_value(),
                output_value
            )


"""mysite URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from django.conf.urls import include, url

from login import views
from game.views import share
urlpatterns = [
    #url(r'^polls/', include('polls.urls')),
    url(r'^admin/', admin.site.urls),
    url(r'^index/', views.index),
    url(r'^login/', views.login),
    url(r'^register/', views.register),
    url(r'^logout/', views.logout),
    url(r'^game/', views.game),
    url(r'^game/(?P<game_id>\d*)/(?P<channel>\S*)', views.game),
    url(r'^white/',views.whiteList),
    url(r'^version/',views.versionCtrl),
    url(r'^share/', share),
    url(r'^captcha', include('captcha.urls'))
]


#!/usr/bin/env python

import sys
import os
import string

for line in sys.stdin:

	attributes = line.split(',')
	summons = attributes[0]
	plate = attributes[14]
	v_precinct = attributes[6]
	v_code = attributes[2]
	date = attributes[1]

	if(len(attributes) != 22):
		plate = attributes[15]

	print('%s\t%s, %s, %s, %s' % (summons, plate, v_precinct, v_code, date))


import requests
from json import loads
from re import findall
from termcolor import colored
from configparser import RawConfigParser

def init(domain):
	MV = []

	print(colored("[*]-Searching Maltiverse...", "yellow"))

	items = 500
	page = 0
	headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"}
	url = "https://api.maltiverse.com/search"

	try:
		while items == 500:
			params = {"query": "hostname.keyword:*.{0}".format(domain), "from": 500*page, "size": 500}
			response = requests.get(url, headers=headers, params=params)
			items = len(loads(response.text)["hits"]["hits"])
			page += 1

			if items > 0:
				results = findall("([\w\d][\w\d\-\.]*\.{0})".format(domain.replace(".", "\.")), response.text)
				MV.extend(results)

		MV = set(MV)

		print("  \__ {0}: {1}".format(colored("Subdomains found", "cyan"), colored(len(MV), "yellow")))
		return MV

	except requests.exceptions.RequestException as err:
		print("  \__", colored(err, "red"))
		return []

	except requests.exceptions.HTTPError as errh:
		print("  \__", colored(errh, "red"))
		return []

	except requests.exceptions.ConnectionError as errc:
		print("  \__", colored(errc, "red"))
		return []

	except requests.exceptions.Timeout as errt:
		print("  \__", colored(errt, "red"))
		return []

	except Exception:
		print("  \__", colored("Something went wrong!", "red"))
		return []


import unittest
from smartystreets_python_sdk import Response, exceptions, Batch
from smartystreets_python_sdk.us_street import Client, Lookup, Candidate
from test.mocks import RequestCapturingSender, FakeSerializer, FakeDeserializer, MockSender, MockExceptionSender

class TestClient(unittest.TestCase):
    def test_freeform_assigned_to_street_field(self):
        lookup = Lookup("freeform address")

        self.assertEqual("freeform address", lookup.street)

    def test_empty_batch_not_sent(self):
        sender = RequestCapturingSender()
        client = Client(sender, None)

        client.send_batch(Batch())

        self.assertIsNone(sender.request)

    def test_successfully_sends_batch(self):
        expected_payload = "Hello, World!"
        sender = RequestCapturingSender()
        serializer = FakeSerializer(expected_payload)
        client = Client(sender, serializer)
        batch = Batch()
        batch.add(Lookup())
        batch.add(Lookup())

        client.send_batch(batch)

        self.assertEqual(expected_payload, sender.request.payload)

    def test_deserialize_called_with_response_body(self):
        response = Response("Hello, World!", 0)
        sender = MockSender(response)
        deserializer = FakeDeserializer(None)
        client = Client(sender, deserializer)

        client.send_lookup(Lookup())

        self.assertEqual(response.payload, deserializer.input)

    def test_candidates_correctly_assigned_to_corresponding_lookup(self):
        candidate0 = {'input_index': 0, 'candidate_index': 0, 'addressee': 'Mister 0'}
        candidate1 = {'input_index': 1, 'candidate_index': 0, 'addressee': 'Mister 1'}
        candidate2 = {'input_index': 1, 'candidate_index': 1, 'addressee': 'Mister 2'}
        raw_candidates = [candidate0, candidate1, candidate2]

        expected_candidates = [Candidate(candidate0), Candidate(candidate1), Candidate(candidate2)]
        batch = Batch()
        batch.add(Lookup())
        batch.add(Lookup())
        sender = MockSender(Response("[]", 0))
        deserializer = FakeDeserializer(raw_candidates)
        client = Client(sender, deserializer)

        client.send_batch(batch)

        self.assertEqual(expected_candidates[0].addressee, batch[0].result[0].addressee)
        self.assertEqual(expected_candidates[1].addressee, batch[1].result[0].addressee)
        self.assertEqual(expected_candidates[2].addressee, batch[1].result[1].addressee)

    def test_raises_exception_when_response_has_error(self):
        exception = exceptions.BadCredentialsError
        client = Client(MockExceptionSender(exception), FakeSerializer(None))

        self.assertRaises(exception, client.send_lookup, Lookup())


import numpy as np

import rlkit.misc.hyperparameter as hyp
from rlkit.launchers.launcher_util import run_experiment
from rlkit.torch.grill.launcher import train_vae

def get_dataset(variant):
    filename = variant['filename']
    data = np.load(filename).item()
    imgs = data['obs']
    split = int(0.8 * len(imgs))
    train_dataset, test_dataset = imgs[:split], imgs[split:]
    return train_dataset, test_dataset, {}

if __name__ == '__main__':
    variant = dict(
        beta=0.5 / 128,
        representation_size=32,
        imsize=84,
        generate_vae_data_fctn=get_dataset,
        save_period=10,
        num_epochs=1000,
        decoder_activation='sigmoid',
        generate_vae_dataset_kwargs=dict(
            filename="/home/vitchyr/git/railrl/data/raw/12"
                     "-29_Image84WheeledCarEnv-v0_N10000__imsize84_oracleTrue.npy",
        ),
        vae_kwargs=dict(
            input_channels=3,
        ),
        algo_kwargs=dict(
            lr=1e-3,
            # use_parallel_dataloading=False,
        ),
    )

    n_seeds = 1
    mode = 'here_no_doodad'
    exp_prefix = 'dev-intro-vae'

    # n_seeds = 3
    # mode = 'ec2'
    # exp_prefix = 'name'

    search_space = {
    }
    sweeper = hyp.DeterministicHyperparameterSweeper(
        search_space, default_parameters=variant,
    )
    for exp_id, variant in enumerate(sweeper.iterate_hyperparameters()):
        for _ in range(n_seeds):
            run_experiment(
                train_vae,
                exp_prefix=exp_prefix,
                mode=mode,
                variant=variant,
                exp_id=exp_id,
                use_gpu=True,
            )


#!/usr/bin/env python
"""
MeerPipe: Processing pipeline for pulsar timing data.

__author__ = "Aditya Parthasarathy"
__copyright__ = "Copyright 2019, MeerTime"
__credits__ = ["Renee Spiewak", "Daniel Reardon"]
__license__ = "Public Domain"
__version__ = "0.2"
__maintainer__ = "Aditya Parthasarathy"
__email__ = "adityapartha3112@gmail.com"
__status__ = "Development"

"""

"""
Contains routines that help initialize the pipeline.
"""

import os
import sys
import shlex
import subprocess
import argparse
import os.path
import numpy as np
import logging
import glob
from shutil import copyfile

def parse_config(path_cfile):
    """
    INPUT: Path to the configuration file
    """

    config_params = {}
    with open (str(path_cfile)) as cfile:
        for line in cfile.readlines():
            sline = line.split("=")
            attr = (sline[0].rstrip())
            if attr == 'input_path':
                config_params["input_path"] = sline[1].rstrip().lstrip(' ')
            if attr == 'output_path':
                config_params["output_path"] = sline[1].rstrip().lstrip(' ')
            if attr == 'flags':
                config_params["flags"] = sline[1].rstrip().lstrip(' ').split(',')
            if attr == 'repo_urls':
                config_params["repos"] = sline[1].rstrip().lstrip(' ').split(',')
            if attr == 'rfi_alg':
                config_params["rfi_alg"] = sline[1].rstrip().lstrip(' ')
            if attr == "project_id":
                config_params["pid"] = sline[1].rstrip().lstrip(' ')
            if attr == "type":
                config_params["type"] = sline[1].rstrip().lstrip(' ')
            if attr == "user":
                config_params["user"] = sline[1].rstrip().lstrip(' ')

    cfile.close()

    return config_params

def setup_logging(path,verbose,file_log):
    """
    Setup log handler - this logs in the terminal (if not run with --slurm).
    For slurm based runs - the logging is done by the job queue system

    """
    log_toggle=False

    # create formatter
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    if file_log == True:
        logfile = "meerpipe.log"
        logger = logging.getLogger(logfile)
        logger.setLevel(logging.INFO)

        if not os.path.exists(path):
            os.makedirs(path)
        #Create file logging only if logging file path is specified
        fh = logging.FileHandler(os.path.join(path,logfile))
        fh.setLevel(logging.INFO)
        fh.setFormatter(formatter)
        logger.addHandler(fh)
        #Check if file already exists, if so, add a demarcator to differentiate among runs
        if os.path.exists(os.path.join(path, logfile)):
            with open(os.path.join(path,logfile), 'a') as f:
                f.write(20*"#")
                f.write("\n")
        logger.info("File handler created")
        log_toggle=True

    if verbose:
        #Create console handler with a lower log level (INFO)
        logfile = "meerpipe.log"
        logger = logging.getLogger(logfile)
        logger.setLevel(logging.INFO)
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)
        ch.setFormatter(formatter)
        # add the handlers to the logger
        logger.addHandler(ch)
        logger.info("Verbose mode enabled")
        log_toggle=True

    """
    #Create console handler with a lower log level (INFO)
    logfile = "meerpipe.log"
    logger = logging.getLogger(logfile)
    logger.setLevel(logging.INFO)
    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(ch)
    logger.info("Verbose mode enabled")
    """

    if log_toggle:
        return logger
    else:
        return none

def get_outputinfo(cparams,logger):
    """
    Routine to gather information about the directory structure from the input data
    The input path is assumed to have directories that are pulsarname/UTCs/beamnumber/freq/*ar
    Returns a list of output paths that needs to exist for the data products to be stored
    """
    input_path = cparams["input_path"]
    output_path = cparams["output_path"]
    logger.info("Gathering directory structure information")
    results_path=[]
    all_archives=[]
    psrnames=[]
    observations = []

    if cparams["batch"] == "batch":
        pulsar_dirs = sorted(glob.glob(os.path.join(input_path,cparams["dirname"])))
    elif cparams["batch"] == "none":
        pulsar_dirs = sorted(glob.glob(cparams["dirname"]))

    if cparams["type"] == "caspsr":
        #For CASPSR data
        pid = str(cparams["pid"])
        for pulsar in pulsar_dirs:
            psr_path,psr_name = os.path.split(pulsar)
            psr_name_split = psr_name.split("_")
            if not psr_name_split[-1] == "R":
                obs_dirs = sorted(glob.glob(os.path.join(pulsar,"*")))
                #logger.info("Number of observations: {0}".format((len(obs_dirs))))
                for num,observation in enumerate(obs_dirs):
                    obs_path,obs_name = os.path.split(observation)
                    if not num > 50:
                        freq_dirs = sorted(glob.glob(os.path.join(observation,"*")))
                        for files in freq_dirs:
                            freq_path,freq_name = os.path.split(files)
                            archives = sorted(glob.glob(os.path.join(files,"*.ar"))) #TODO: change extension to be generic
                            results_path.append(str(output_path+"/"+pid+"/"+psr_name+"/"+obs_name+"/"+freq_name+"/"))
                            psrnames.append(psr_name)
                            all_archives.append(archives)
                    else:
                        logger.info("Number of observations exceeded 50")
                        break
            else:
                print ("Skipping CAL observations : {0}".format(psr_name))

    elif cparams["type"] == "ppta_zap":
        #For PPTA zapping
        for pulsar in pulsar_dirs:
            psr_path,psr_name = os.path.split(pulsar)
            psrnames.append(psr_name)
            archives = sorted(glob.glob(os.path.join(pulsar,"*.dzF"))) #TODO: change extension to be generic
            results_path.append(str(output_path+"/"+psr_name+"/"))
            all_archives.append(archives)

    elif cparams["type"] == "meertime":
        #For MeerTime data on OzStar
        #Example: 2018-10-25-05:31:38/J0835-4510/1284/<*.ar>
        #New directory structure: <psrname>/<UTC>/<beam>/<freq>/<*.ar>
        """
        for observation in pulsar_dirs:
            obs_path,obs_name = os.path.split(observation)
            pulsars = sorted(glob.glob(os.path.join(observation,"J*")))
            logger.info("{0}".format(pulsars))
            for pulsar in pulsars:
                psr_path,psr_name = os.path.split(pulsar)
                psr_name_split = psr_name.split("_")
                if not psr_name_split[-1] == "R":
                    freq_dirs = sorted(glob.glob(os.path.join(pulsar,"*")))
                    for files in freq_dirs:
                        freq_path,freq_name = os.path.split(files)
                        archives = sorted(glob.glob(os.path.join(files,"*.ar")))
                        results_path.append(str(output_path+"/"+obs_name+"/"+psr_name+"/"+freq_name+"/"))
                        psrnames.append(psr_name)
                        all_archives.append(archives)
        """

        #NEW directory structure
        for pulsar in pulsar_dirs:
            psr_path,psr_name = os.path.split(pulsar)
            psr_name_split = psr_name.split("_")
            if not psr_name_split[-1] == "R":
                observation_dirs = sorted(glob.glob(os.path.join(pulsar,"2*")))
                if not cparams["utc"] == "none":
                    logger.info("Processing custom UTC")
                    observation_dirs = glob.glob(cparams["utc"])

            for observation in observation_dirs:
                obs_path,obs_name = os.path.split(observation)
                beam_dirs = sorted(glob.glob(os.path.join(observation,"*")))
                for beam in beam_dirs:
                    beam_path,beam_name = os.path.split(beam)
                    freq_dirs = sorted(glob.glob(os.path.join(beam,"*")))
                    logger.info("{0}".format(freq_dirs))
                    for files in freq_dirs:
                        freq_path,freq_name = os.path.split(files)
                        archives = sorted(glob.glob(os.path.join(files,"*.ar")))
                        results_path.append(str(output_path+"/"+psr_name+"/"+obs_name+"/"+beam_name+"/"+freq_name+"/"))
                        psrnames.append(psr_name)
                        all_archives.append(archives)

    return results_path,all_archives,psrnames

def create_structure(output_dir,cparams,psrname,logger):
    """
    Routine to create an output directory structure as decided by get_directoryinfo.
    Creates a "cleaned", "calibrated" and a "timing" directory in each output path.

    Now includes a "scintillation" directory.
    """
    output_path = cparams["output_path"]
    repos = cparams["repos"]
    flags = cparams["flags"]

    logger.info("Creating the directory structure for {0}".format(psrname))
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    cleaned_dir = os.path.join(output_dir,"cleaned")
    calibrated_dir = os.path.join(output_dir,"calibrated")
    timing_dir = os.path.join(output_dir,"timing")
    decimated_dir = os.path.join(output_dir,"decimated")
    scintillation_dir = os.path.join(output_dir,"scintillation")

    if cparams["type"] == "caspsr":
        #Creating the Project ID directory
        project_dir = os.path.join(output_path,str(cparams["pid"]))
        #NOTE: Pulsar directory is the main directory containing the global par and tim files and the processed obs directories.
        pulsar_dir = os.path.join(project_dir,psrname)
    elif cparams["type"] == "meertime":
        pulsar_dir = output_dir
        project_dir = output_dir
    elif cparams["type"] == "ppta_zap":
        pulsar_dir = output_dir
        project_dir = output_dir

    if not os.path.exists(cleaned_dir):
        logger.info("Cleaned directory created")
        os.makedirs(cleaned_dir)
    else:
        logger.info("Cleaned directory exists")

    if not os.path.exists(calibrated_dir):
        logger.info("Calibrated directory created")
        os.makedirs(calibrated_dir)
    else:
        logger.info("Calibrated directory exists")

    if not os.path.exists(timing_dir):
        logger.info("Timing directory created")
        os.makedirs(timing_dir)
    else:
        logger.info("Timing directory exists")

    if not os.path.exists(decimated_dir):
        logger.info("Decimated directory created")
        os.makedirs(decimated_dir)
    else:
        logger.info("Decimated directory exists")

    if not os.path.exists(scintillation_dir):
        logger.info("Scintillation directory created")
        os.makedirs(scintillation_dir)
    else:
        logger.info("Scintillation directory exists")

    if not os.path.exists(pulsar_dir):
        logger.info("Pulsar directory created")
        os.makedirs(pulsar_dir)
    else:
        logger.info("Pulsar directory exists")

    if not os.path.exists(project_dir):
        logger.info("Project directory created")
        os.makedirs(project_dir)
    else:
        logger.info("Project directory exists")

    #Pull/Update repositories
    #TODO: for now just creating directories. Have to manage_repos eventually!
    logger.info("Checking if the ephemerides and templates directory exists")
    if not os.path.exists(os.path.join(output_path,"meertime_ephemerides")):
        logger.info("Ephemerides directory created")
        os.makedirs(os.path.join(output_path,"meertime_ephemerides"))
    else:
        logger.info("meertime_ephemeredis exists")

    if not os.path.exists(os.path.join(output_path,"meertime_templates")):
        logger.info("Templates directory created")
        os.makedirs(os.path.join(output_path,"meertime_templates"))
    else:
        logger.info("meertime_templates exists")

    #Check for the pulsar epehemeris and templates and copy them to the pulsar directory
    ephem_dir = os.path.join(output_path,"meertime_ephemerides")
    template_dir = os.path.join(output_path,"meertime_templates")
    #Copying pulsar ephemeris
    if os.path.exists(os.path.join(ephem_dir,psrname+".par")):
        logger.info("Ephemeris for {0} found".format(psrname))
        copyfile(os.path.join(ephem_dir,psrname+".par"),os.path.join(pulsar_dir,psrname+".par"))
    else:
        logger.info("Ephemeris for {0} not found. Generating new one.".format(psrname))
        psrcat = "psrcat -all -e {0}".format(psrname)
        proc = shlex.split(psrcat)
        f = open("{0}/{1}.par".format(ephem_dir,psrname),"w")
        subprocess.call(proc,stdout=f)
        logger.info("An ephemeris was generated from the psrcat database")
        copyfile(os.path.join(ephem_dir,psrname+".par"),os.path.join(pulsar_dir,psrname+".par"))

    #Copying pulsar template
    if os.path.exists(os.path.join(template_dir,psrname+".std")):
        logger.info("Template for {0} found".format(psrname))
        copyfile(os.path.join(template_dir,psrname+".std"),os.path.join(pulsar_dir,psrname+".std"))
    else:
        #TODO:Generate new template in this case
        logger.info("Template for {0} not found. Will generate one after zapping.".format(psrname))

        #sys.exit()