'''
	File: 	profile_generator.py
	Author:	Nicholas Mattei (nicholas.mattei@nicta.com.au)
	Date:	Sept 11, 2013
			November 6th, 2013
			July 30th, 2014

  * Copyright (c) 2014, Nicholas Mattei and NICTA
  * All rights reserved.
  *
  * Developed by: Nicholas Mattei
  *               NICTA
  *               http://www.nickmattei.net
  *               http://www.preflib.org
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *     * Redistributions of source code must retain the above copyright
  *       notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above copyright
  *       notice, this list of conditions and the following disclaimer in the
  *       documentation and/or other materials provided with the distribution.
  *     * Neither the name of NICTA nor the
  *       names of its contributors may be used to endorse or promote products
  *       derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY NICTA ''AS IS'' AND ANY
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL NICTA BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

About
--------------------
	This file generates voting profiles according to a given distribution.
	It requires PreflibUtils to work properly.

NOTE
--------------------
	This is a heavily modified version of this file which does not
	require the use of PrefLibUtils for use only with the Impartial
	experiments.

'''
import random
import itertools
import math
import copy
import argparse
import sys
import collections
from scipy import stats
import numpy as np

_DEBUG = False

def generate_approx_m_regular_assignment(agents, m, clusters={}, randomize=True):
  """
  Creates an "approximatly" m regular reviewing assignment
  for all agents subject to the restriction that no one
  reviews themselves or anyone in their cluster.  This is
  approximate as it will balance as much as possible
  but not be exact.  We attempt to do this in a cyclical manner;
  this algorithm will fail if each person is asked to review
  too many people or the clusters are not balanced.

  Parameters
  -----------
  agents: array like
  	list of agents to be reviewed.

  m: integer
  	Number of elements that each agent should review.  This does
  	not mean the number of reviews that each agent will recieve.

  clusters: dict
    A mapping from integer ---> [agents] where each agent in an
    partition together.  Agents should not review other agents
    in their own partition.  Also, partitions must be disjoint.

  Returns
  -----------
  assignment: dict
  	A dict from agent --> [agents] which is a list of lenght m
  	of agents not in the partition of agent i.

  Notes
  -----------
  """
  #Verify the clusters...
  if clusters != {}:
  	if _DEBUG: print("\nClusters:\n" + str(clusters))
  	#Ensure that the partitions don't overlap.
  	agent_set = list(itertools.chain(*clusters.values()))
  	if len(agent_set) != len(set(agent_set)):
  		print("clustering contains duplicates in different clusters")
  		return 0
  else:
  	#Make everone their own cluster if we don't have a clustering.
  	clusters = {i:[agents[i]] for i in range(len(agents))}
  	if _DEBUG: print("\nClusters:\n" + str(clusters))

  # Do a check here -- if, for every cluster, the number of agents
  # outside is < m fail, can't review 2x.
  if any([m > len(agents) - len(ci) for k,ci in clusters.items()]):
    print("m is larger than N - Ci for some Ci, duplicate review required.")
    return 0

  cluster_assignment = {}
  # Shuffling a dequq scales as n^2, faster to copy..
  t = list(clusters.keys())
  if randomize:
    random.shuffle(t)
  cluster_deq = collections.deque(t)

  for j in clusters.keys():
    current = cluster_deq.popleft()
    c_list = list(cluster_deq)
    assn = c_list*math.ceil(m*len(clusters[current]) / len(c_list))
    assn = assn[:m*len(clusters[current])]
    cluster_assignment[current] = assn
    cluster_deq.append(current)

  if _DEBUG: print("Cluster Assignment: " + str(cluster_assignment))

  agent_to_clusters = {k:[] for k in agents}
  # Iterate over a cluster --> cluster assignment and convert it
  # to a agent --> cluster assignement using a canoical ordering
  # for the agents in each cluster on the RHS.
  for c_cluster in cluster_assignment.keys():
    c_agents = copy.copy(clusters[c_cluster])
    if randomize:
      random.shuffle(c_agents)
    targets = cluster_assignment[c_cluster]
    for c_a in c_agents:
      agent_to_clusters[c_a] = targets[:m]
      targets = targets[m:]

  if _DEBUG: print("Agents to Clusters: " + str(agent_to_clusters))

  # Convert the RHS.  For every agent, replace a agent --> cluster
  # assignment to a agent --> agent assignment.  Use a canoical
  # ordering of the agents on the RHS randomized if necessary.

  agent_assignment = {k:[] for k in agents}
  target_order = {}
  # Build RHS ordering.
  for k,v in clusters.items():
    t = copy.copy(v)
    if randomize:
      random.shuffle(t)
    target_order[k] = collections.deque(t)

  for a, t in agent_to_clusters.items():
    # For each target cluster
    for cc in t:
      #Assign the front of the list and then rotate it
      agent_assignment[a].append(target_order[cc][0])
      target_order[cc].rotate(-1)

  # Post check for duplicates..
  for k,v in agent_assignment.items():
    if len(v) != len(set(v)):
      print("Double review assignment: ", str(k), " :: ", str(v))
    if len(v) != m:
      print("Error in assignment, agent ", str(k), " has less than m reviews ", str(v))
  if _DEBUG: print("Agent to Agent: " + str(agent_assignment))

  return agent_assignment

def generate_mallows_mixture_profile(voters, candidates, distribution, reference_rankings, phis):
  """
  For every voter generates an ordinal preference profile over the
  set of candidates according to a mallows mixture model
  parameterized by a distribution over mallows models which
  consist of a reference ranking and a dispersion parameter (phi)

  Parameters
  -----------
  voters: array like

  candidates: array like

  distribution: array like
    A distribution over discrete mallows models, must sum to 1.0.

  reference_rankings: array like
    Same lenght as distribution, must be a full ranking over candidates.

  phis: array like of floats
    Same length as distribtion, must be between 0.0 and 1.0 and describes
    the dispersion parameter for the indexed model.

  Returns
  -----------
  profile: dict
    A mapping from voters ---> [ranking] where the complete element
    set of candidats is ordered in ranking with ranking[0] being
    the most prefered element.

  Notes
  -----------
  Generate a Mallows model with the various mixing parameters passed.
  Distribution should be a mix over the models that sums to ~1.0.
  phis is an array len(phis) = len(mix) = len(refs) that is the phi for the particular models

  Method mostly implemented from Lu, Tyler, and Craig Boutilier.
  "Learning Mallows models with pairwise preferences."
  Proceedings of the 28th International Conference on Machine Learning (
  ICML-11). 2011.

  """
  if not np.isclose(sum(distribution), 1.0):
    print("Mallows distribution isn't close to 1.0")
    return 0
  if len(distribution) != len(reference_rankings) or len(reference_rankings) != len(phis):
    print("Insufficient number of parameters for mallows mix.")
    return 0

  # Using Numpy discrete and precomputing the models has better speed for large N.
  # Using RVS in this way seems to scale much better.
  #       100, 1000, 2000
  # draw   <2,   94,   724s
  # RVS    <2,   92,   356s
  # Numpy RV version...

  # Define the RVS for the draw over distros...
  model_rvs = stats.rv_discrete(values=(list(range(len(distribution))),distribution))

  insertion_rvs = {}
  # For each of the models we need to precompute the insertion distributions and
  # associated RVs.
  for m,p in enumerate(phis):
    # Returns an insertion probability vector  according to phi.
    # this is the probability that ref[i-1] should be inserted into position []
    insertion_distribution = compute_mallows_insertion_distribution(len(candidates), p)

    # Make a set of RV's for each element.
    model_insertion_rvs = {}
    for i in insertion_distribution.keys():
      model_insertion_rvs[i] = stats.rv_discrete(values=(list(range(len(insertion_distribution[i]))), insertion_distribution[i]))
    insertion_rvs[m] = model_insertion_rvs

  profile = {}
  for c_voter in voters:
    # Draw a model.
    c_model = model_rvs.rvs()
    # Will generate a list (strict order) over the elements of candidates.
    profile[c_voter] = []
    for i,c in enumerate(reference_rankings[c_model]):
      profile[c_voter].insert(insertion_rvs[c_model][i].rvs(), c)

  return profile

def compute_mallows_insertion_distribution(length, phi):
  """
  Helper function for the mallows distro above.
  For Phi and a given number of candidates, compute the
  insertion probability vectors.

  Method mostly implemented from Lu, Tyler, and Craig Boutilier.
  "Learning Mallows models with pairwise preferences."
  Proceedings of the 28th International Conference on Machine Learning (
  ICML-11). 2011.

  Parameters
  -----------
  length: integer
    Number of insertion vectors to compute.  Equal to the number of
    items that the distribution is over.

  phi: float
    Dispersion parameter.  0.0 gives a unanimous culture while
    1.0 gives the impartial culture.

  Returns
  -----------
  vec_dist: dict
    A mapping from index ---> insertion vector
    where each element of the insertion vector (insert[i]) is the probability
    that candidate[index] is inserted at insert[i].  Hence the the length
    of insert is equal to index+1 (so 1, 2, 3 ....)

  Notes
  -----------
  """
  # For each element in length, compute an insertion
  # probability distribution according to phi.
  vec_dist = {}
  for i in range(length):
    # Start with an empty distro of length i+1
    dist = [0] * (i+1)
    #compute the denom = phi^0 + phi^1 + ... phi^(i-1)
    denom = sum([pow(phi,k) for k in range(len(dist))])
    #Fill each element of the distro with phi^i-j / denom
    for j in range(len(dist)):
      dist[j] = pow(phi, i - j) / denom
      #print(str(dist) + "total: " + str(sum(dist)))
      vec_dist[i] = dist
  return vec_dist

def remove_candidates(orders, candidates_to_remove):
  """
    Remove a set of candidates from a list representing a preference order.
  """
  projection = []
  for c_vote in orders:
    temp_vote = copy.copy(c_vote)
    for c_remove in candidates_to_remove:
      temp_vote.remove(c_remove)
    projection.append(temp_vote)
  return projection

def restrict_score_matrix(score_matrix, assignment):
  """
  Given a score matrix and an assignment, change
  all the elements of the score matrix not present in
  the assignment to 0.

  Parameters
  -----------
  score_matrix: array like
    The numerical scores of the agents for all the other agents.
    We use the convention that a[i,j] is the grade that agent
    j gives to i.  This means that column j is all the grades
    *given* by agent j and that row i is the grades *recieved*
    by agent i.

  assignment: dict
    A dict from agent --> [agents] which is a list of lenght m
    of agents not in the partition of agent i.

  Returns
  -----------
  restricted_score_matrix: array like
    A copy of the score matrix passed in with every element
    not in the assignment removed.

  Notes
  -----------
  This just copies over the valid entries and leaves the rest as zero.
  Be careful when mixing with the normzliation code as it could lead to
  some strange ass shit...

  """

  restricted_score_matrix = np.zeros((len(assignment.keys()), len(assignment.keys())))

  for c_agent, c_assignment in assignment.items():
    for i in c_assignment:
      restricted_score_matrix[i][c_agent] = score_matrix[i][c_agent]

  return restricted_score_matrix

def profile_utilities_to_score_matrix(profile, utilities):
  """
  Given a profile and a COMMON! utility vector, create a score matrix.

  Parameters
  -----------
  profile: dict
    mapping of agents to their orders as a list with list[0] being
    the most prefered.

  utilities: array like
    list of utility mapped to the index of the profile.  this vector must
    be at least as long as the longest order.

  Returns
  -----------
  score_matrix: array like
    The numerical scores of the agents for all the other agents.
    We use the convention that a[i,j] is the grade that agent
    j gives to i.  This means that column j is all the grades
    *given* by agent j and that row i is the grades *recieved*
    by agent i.

  Notes
  -----------
  """
  #Verify that the utility vector is long enough...
  for agent,c_vote in profile.items():
    if len(c_vote) > len(utilities):
      print("Agent vote is longer than utility vector")
      return 0

  score_matrix = np.zeros((len(profile.keys()), len(profile.keys())))
  # Otherwise, build a column element...
  for c_agent in profile.keys():
    c_column = np.zeros(len(profile.keys()))
    for i, c_other in enumerate(profile[c_agent]):
      c_column[c_other] = utilities[i]
    score_matrix[:, c_agent] = c_column

  return score_matrix

def profile_classes_to_score_matrix(profile, scores, distribution = []):
  """
  Given a profile and vector of scores, block the candidates
  into size 1/len(score) blocks and assign everyone in that block
  that value.

  Parameters
  -----------
  profile: dict
    mapping of agents to their orders as a list with list[0] being
    the most prefered.

  scores: array like
    score[i] is the score assigned to the fraction 1/len(score) of
    the agents.  So score[0] is assigned to the first n/len(score)
    agents in my profilea and on.  Will use ceil so it's well defined.

  distribution: array like
    a vector of length scores which defines the width of that
    score distribution.  Implicitly the score[0] has width
    equal to the first distribution[0] portion of the profile.

  Returns
  -----------
  score_matrix: array like
    The numerical scores of the agents for all the other agents.
    We use the convention that a[i,j] is the grade that agent
    j gives to i.  This means that column j is all the grades
    *given* by agent j and that row i is the grades *recieved*
    by agent i.

  Notes
  -----------
  The way this is written someone gets every score if possible.  I.e. if score[0]
  and score[1] should go to the top guy only, then the 2nd place guy gets score[1].
  Don't submit stupid score vectors (e.g., scores=[1000, 5000, 0], distribution=[0.0000001, 0.000001, 1-0.0000002])
  as the top and aecond top guy will get 1000 and 5000 respectivly.

  """
  #Verify that the utility vector is long enough...
  if len(scores) <= 1:
    print("Scores vector is <= 1... cannot use.")
    return 0

  score_matrix = np.zeros((len(profile.keys()), len(profile.keys())))

  if distribution != []:
    if not np.isclose(sum(distribution), 1.0):
      print("Distribution of scores is not a distribution")
      return 0
    if len(scores) != len(distribution):
      print("Distribution and scores are different lenghts!")
      return 0

    if _DEBUG: print("Score: ", str(scores))

    # Otherwise, build a column element...
    for c_agent in profile.keys():
      #Find the right column...
      c_column = np.array([0]*len(profile.keys()))
      for i, c_other in enumerate(profile[c_agent]):
        #Determine the \leq index.
        cv = 0
        p = float(i) / float(len(profile[c_agent])) - distribution[cv]
        while p > 0.0:
          cv+=1
          p -= distribution[cv]
        c_column[c_other] = scores[cv]
      score_matrix[:, c_agent] = c_column

  # Use even distribution...
  else:
    if _DEBUG: print("Score: ", str(scores))
    # Otherwise, build a column element...
    for c_agent in profile.keys():
    # If index(c_agent) <= n/len(score)
    # points(c_agent) = score[ FLOOR((index(c_agent) / n) / (1./len(score)))  ]
    # Maps percentile to index of score function... hence 1/s% --> 0, 1/(s+1 -- 1/2s --> 1)...
      c_column = np.array([0]*len(profile.keys()))
      for i, c_other in enumerate(profile[c_agent]):
        c_column[c_other] = scores[int(math.floor((float(i)/ float(len(profile[c_agent])))  * float(len(scores))))]
      score_matrix[:, c_agent] = c_column

  return score_matrix

def strict_m_score_matrix(profile, assignment, scores):
  """
  Given a score matrix and an assignment, change
  all the elements of the score matrix not present in
  the assignment to 0.  Also score those present
  in the assignment strictly according to the scores
  with score[0] being applied to the most prefered agent.

  Parameters
  -----------
  profile: dict
    mapping of agents to their orders as a list with list[0] being
    the most prefered.

  assignment: dict
    A dict from agent --> [agents] which is a list of lenght m
    of agents not in the partition of agent i.

  scores: array like
    An array of length m which is the points assigned to
    the most prefered agent to the least prefered agent.

  Returns
  -----------
  restricted_score_matrix: array like
    The numerical scores of the agents for their assigned agents
    according to the scores given.

    We use the convention that a[i,j] is the grade that agent
    j gives to i.  This means that column j is all the grades
    *given* by agent j and that row i is the grades *recieved*
    by agent i.

  Notes
  -----------
  This just copies over the valid entries and leaves the rest as zero.
  Be careful when mixing with the normzliation code as it could lead to
  some strange ass shit...

  """
  #Verify that the utility vector is long enough...
  for k,v in assignment.items():
    if len(v) > len(scores):
      print("Agent assignment is longer than score vector")
      print("Agent " + str(k) + " Assignment: " + str(v) + " Scores: " + str(score))
      exit()

  strict_m_matrix = np.zeros((len(profile.keys()), len(profile.keys())))
  # Otherwise, build a column element...
  for c_agent in profile.keys():
    c_column = np.zeros(len(profile.keys()))
    # Sorting tuple is (index, agent)
    p = []
    for i,o in enumerate(profile[c_agent]):
      if o in assignment[c_agent]:
        p.append((i, o))
    # p is now the sorted list of the assignment.
    p = sorted(p, key=lambda x:x[0], reverse=False)
    for i,t in enumerate(p):
      c_column[t[1]] = scores[i]
    strict_m_matrix[:, c_agent] = c_column

  return strict_m_matrix

  strict_m_matrix = np.zeros((len(assignment.keys()), len(assignment.keys())))

  restricted = restrict_score_matrix(score_matrix, assignment)

  print(restricted)

  return strict_m_matrix


# Make sure you are using the following environment to run this shell
# Python 3.5.2
# Requests 2.12.4

import requests
import re

url = 'http://www.pythonchallenge.com/pc/def/linkedlist.php'
parameters = {"nothing": 12345}
pattern = re.compile(r'[0-9]+$')
new_number = 0

# while parameters['nothing'] != new_number:
while True:
    ret = requests.get(url, params = parameters)
    res = pattern.search(ret.text)
    if not res:
        if ret.text == "Yes. Divide by two and keep going.":
            parameters['nothing'] = int(parameters['nothing']) / 2
            continue
        else:
            print(ret.text) # well, the result here is "peak.html"
            break
    new_number = res.group(0)
    parameters['nothing'] = new_number


from collections import deque

for _ in range(10):
    t = int(input())

    arr = deque(map(int, input().split()))

    while 1:
        for i in range(1,6):
            a = arr.popleft()
            if (a-i) <= 0:
                arr.append(0)
                print('#%d' %(t), end=' ')
                for j in arr:
                    print(j, end= ' ')
                print()
                break
            else:
                arr.append(a-i)
        if arr[-1] == 0:
            break

python daily coding/2020.6.25 (SW Expert Academy)/1225번 (암호생성기).py


from bs4 import BeautifulSoup
import requests
import pandas as pd

result = {'date' : [],
          'title' : [],
          'content' : []}

print('수집 시작')

for j in range(2):
    p = requests.get('https://www.inven.co.kr/board/lostark/5352?p=' + str(j))
    soup = BeautifulSoup(p.text, 'html.parser')

    selection = soup.select('a.subject-link')

    for i in range(len(selection)):
        title = selection[i].text.replace("\xa0[밸런스]\xa0\xa0","")
        url = selection[i].get('href')

        q = requests.get(url)
        soup2 = BeautifulSoup(q.text, 'html.parser')

        content = soup2.select('div#powerbbsContent')[0].text
        date = soup2.select('div.articleDate')[0].text

        result['date'].append(date)
        result['title'].append(title)
        result['content'].append(content)

print('수집 완료')

final = pd.DataFrame(result)
final.to_excel('balance_text.xlsx', index = False)

print('파일 저장 완료')


#!/usr/bin/python3

import math
import matplotlib.pyplot as plt
import numpy as np

# Binary entropy function
def H2(x):
    return -x * np.log2(x) - (1-x)*np.log2(1-x)

# Approx inverse
invH = {}
for i in range(499):
    y = float(i)/1000
    x = round(H2(y), 3)
    if math.isnan(x):
        continue
    invH[x] = y
    print(x, y)

def iH2(x):
    closest = min(invH.keys(), key=lambda y:abs(y-x))
    return invH[closest]

# Ternary entropy function
def H3(x):
    q = 3
    return -x * np.log2(x) - (1-x)*np.log2(1-x) + x*np.log2(q-1)
    #return -x * np.log2(x) - (1-x)*np.log2(1-x) + x
    #q = 3
    #return x*math.log(q-1, q) - x*math.log(x, q) - (1-x)*math.log(1-x, q)

# Approx inverse
invH3 = {}
for i in range(499):
    y = float(i)/1000
    try:
        x = round(H3(y), 3)
    except:
        continue
    if math.isnan(x):
        continue
    invH3[x] = y
    print(x, y)

def iH3(x):
    closest = min(invH3.keys(), key=lambda y:abs(y-x))
    return invH3[closest]

# Pentary entropy function
def H5(x):
    q = 5
    return -x * np.log2(x) - (1-x)*np.log2(1-x) + x*np.log2(q-1)
    #return -x * np.log2(x) - (1-x)*np.log2(1-x) + x
    #return x*math.log(q-1, q) - x*math.log(x, q) - (1-x)*math.log(1-x, q)

# Approx inverse
invH5 = {}
for i in range(499):
    y = float(i)/1000
    try:
        x = round(H5(y), 3)
    except:
        continue
    if math.isnan(x):
        continue
    invH5[x] = y
    print(x, y)

def iH5(x):
    closest = min(invH5.keys(), key=lambda y:abs(y-x))
    return invH5[closest]

def get_bound_binary(n, p_range):
    payload=[]
    efficiency=[]
    print("-- n:", n, "--")
    for p in p_range:
        alpha = p*np.log2(n) / ( (n**p-1)/(n-1) )
        y = iH2(alpha)
        e = alpha/y

        if round(alpha,2) <= 0.02:
            continue

        print(p, round(alpha,2), y, round(e,2))

        alpha = 1/alpha
        payload.append(alpha)
        efficiency.append(e)

    return payload, efficiency

def get_bound_ternary(n, p_range):
    payload=[]
    efficiency=[]
    print("-- n:", n, "--")
    for p in p_range:
        alpha = p*np.log2(n) / ( (n**p-1)/(n-1) )
        y = iH3(alpha)
        e = alpha/y

        if round(alpha,2) <= 0.02:
            continue

        print(p, round(alpha,2), y, round(e,2))

        alpha = 1/alpha
        payload.append(alpha)
        efficiency.append(e)

    return payload, efficiency

def get_bound_pentary(n, p_range):
    payload=[]
    efficiency=[]
    print("-- n:", n, "--")
    for p in p_range:
        alpha = p*np.log2(n) / ( (n**p-1)/(n-1) )
        y = iH5(alpha)
        e = alpha/y

        if round(alpha,2) <= 0.00:
            continue

        print(p, round(alpha,2), y, round(e,2))

        alpha = 1/alpha
        payload.append(alpha)
        efficiency.append(e)

    return payload, efficiency

def get_series(n, p_range):
    payload=[]
    efficiency=[]
    print("-- n:", n, "--")
    for p in p_range:
        alpha = p*np.log2(n) / ( (n**p-1)/(n-1) )
        alpha = 1/alpha
        e = p*np.log2(n) / (1-n**(-p) )

        print(p, round(alpha,2), round(e,2))
        payload.append(alpha)
        efficiency.append(e)
    return payload, efficiency

payload, efficiency = get_series(2, range(1,15))
plt.plot(payload, efficiency, label='n=2')

payload, efficiency = get_series(3, range(1,10))
plt.plot(payload, efficiency, label='n=3')

payload, efficiency = get_series(5, range(1,7))
plt.plot(payload, efficiency, label='n=5')

payload, efficiency = get_series(7, range(1,6))
plt.plot(payload, efficiency, label='n=7')

payload, efficiency = get_bound_binary(2, range(1,10))
plt.plot(payload, efficiency, label='binary bound', linestyle='dashed')

payload, efficiency = get_bound_ternary(3, range(1,10))
plt.plot(payload, efficiency, label='ternary bound', linestyle='dashed')

payload, efficiency = get_bound_pentary(5, range(1,10))
plt.plot(payload, efficiency, label='pentary bound', linestyle='dashed')

plt.xlim(0, 20)
plt.xlabel(r'Payload $\alpha^{-1}$')
plt.ylabel('Efficiency')
plt.title('n-ary codes')
plt.grid(True)
plt.legend()

plt.show()


import web
# the urls parts makes the link between "/" and the "index" class
urls = (
"/","index"
)

app = web.application(urls,globals())

render = web.template.render('templates/')

class index:
# GET(self) function is a default function to look for event handle when using web.py
    def GET(self):
        greeting = "Waaagahhhh!"
        # the (greeting = greeting) is given the local greeting var in this .py script to the greeting argument in the template
        # when change the line to render.index(greeting = None),the webpage would display the default message
        return render.index(greeting = greeting)

if __name__ == "__main__":

    app.run()


import math
import win32con, win32api,os
import json
# from PyQt5.QtCore import pyqtSlot
# from PyQt5.QtWidgets import *
# from PyQt5.uic import loadUi

def decryptMessage(fileName, path):
        """
        Function to decrypt the users info and write the result in a text file
        """
        key = 8
        try :
            with open(path+''+fileName+'.txt','r') as json_file:
                usersInfo = json_file.readlines()
                for userInfo in usersInfo:

                    # The transposition decrypt function will simulate the "columns" and
                    # "rows" of the grid that the plaintext is written on by using a list
                    # of strings. First, we need to calculate a few values.
                    # The number of "columns" in our transposition grid:
                    numOfColumns = math.ceil(len(userInfo) / key)
                    # The number of "rows" in our grid will need:
                    numOfRows = key
                    # The number of "shaded boxes" in the last "column" of the grid:
                    numOfShadedBoxes = (numOfColumns * numOfRows) - len(userInfo)
                    # Each string in plaintext represents a column in the grid.
                    plaintext = [''] * numOfColumns
                    # The col and row variables point to where in the grid the next
                    # character in the encrypted message will go.
                    col = 0
                    row = 0
                    for symbol in userInfo:
                        plaintext[col] += symbol
                        col += 1 # point to next column
                        # If there are no more columns OR we're at a shaded box, go back to
                        # the first column and the next row.
                        if (col == numOfColumns) or (col == numOfColumns - 1 and row >= numOfRows - numOfShadedBoxes):
                            col = 0
                            row += 1
            fileName = "decryptInfo.txt"
            with open (path+""+fileName, "w") as decryptInfo:
                # write the decrypted info in a text file
                decryptInfo.write(''.join(plaintext))

            return True

        except Exception as e:
            print(e)
            return False

def getActualUserName():
        """
        Function to get the actual username, return actualUserName
        """
        with open("Session/actualUser.txt", "r") as reader:
            data = reader.readlines()
            for line in data:
                actualUserName = line
        return actualUserName

def encryptInfo(userInfo):
    """
    Function to encrypt users info, take user info as string and return an encrypting string
    """
    key = 8
    # Each string in ciphertext represents a column in the grid.
    ciphertext = [''] * key
    # Loop through each column in ciphertext.
    for col in range(key):
        pointer = col
        # Keep looping until pointer goes past the length of the message.
        while pointer < len(userInfo):
            # Place the character at pointer in message at the end of the
            # current column in the ciphertext list.
            ciphertext[col] += userInfo[pointer]
            # move pointer over
            pointer += key
    # Convert the ciphertext list into a single string value and return it.
    return ''.join(ciphertext)

def printUserAccountInfo(self,username):
        """
        Function to print all info of user
        """
        #list all files name: return an object of type list
        listOfFileName = os.listdir("Save/Account/"+username)
        path = "Save/Account/"+username+"/"
        row = 0
        for fileName in listOfFileName:
            column = 0
            accountName = fileName.split(".")[0]
            if accountName == "decryptInfo":
                pass
            else:
                response = decryptMessage(accountName, path)
                if response == True:
                    with open(path+'decryptInfo.txt') as json_file:
                        # load the content of decryptInfon file
                        data = json.load(json_file)
                        # read the content
                        self.AccountInfo.setRowCount(len(listOfFileName)-1)
                        for infoOf in data[accountName]:
                            self.AccountInfo.setItem(row,column, QTableWidgetItem(infoOf['AccountName']))
                            column += 1
                            self.AccountInfo.setItem(row,column, QTableWidgetItem(infoOf['Email']))
                            column += 1
                            self.AccountInfo.setItem(row,column, QTableWidgetItem(infoOf['Password']))
                            column += 1
                    row += 1
                else :
                    print("erreur")


#-*- coding: UTF-8 -*-
#查询器
import time
from selenium.webdriver.support.select import Select
from WebServiceLayer.ticketelement.ticketWait import now_show,not_show

class querierlist():
    def add_querier(self,driver,customname,queriername,querierdescription):
        driver.find_element_by_css_selector()
        driver.find_element_by_xpath("//a[contains(text(),'工单查询器')]").click()
        #进入iframe id="rightContentIfr"
        driver.switch_to.frame(driver.find_element_by_css_selector("iframe#rightContentIfr"))
        time.sleep(2)
        not_show(driver,"[class='glyphicon glyphicon-plus']")
        driver.find_element_by_css_selector("[class='glyphicon glyphicon-plus']").click()
        text5=driver.find_element_by_css_selector("[class='glyphicon glyphicon-plus']").text
        print(text5)
        now_show(driver,"[name='ticketName']")
        now_show(driver,"[name='ticketDescription']")
        driver.find_element_by_css_selector("[name='ticketName']").click()
        driver.find_element_by_css_selector("[name='ticketName']").send_keys(queriername)
        driver.find_element_by_css_selector("[name='ticketDescription']").click()
        driver.find_element_by_css_selector("[name='ticketDescription']").send_keys(querierdescription)
        now_show(driver,"[id='RequiredPOVItem']")
        #判断满足任意以下条件中有添加的文本字段
        driver.find_element_by_css_selector("[name='RequiredPSelect[0]']").click()
        s=driver.find_element_by_css_selector("[name='RequiredPSelect[0]']")
        Select(s).select_by_visible_text(customname)#通过文本值定位
        #[name="OptionalPSelect[0]"]
        #判断满足任意以下条件中有添加的文本字段
        s = driver.find_element_by_css_selector("[name='OptionalPSelect[0]']")
        Select(s).select_by_visible_text(customname)  # 通过文本值定位
        #等待元素出现#form-group_container1 > div:nth-last-child(1)
        target_elem1 = driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)")
        driver.execute_script("return arguments[0].scrollIntoView();", target_elem1)
        #获取最后文本的值是什么
        wenbenshuju=driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)>span:nth-child(2)").text
        try:
            assert wenbenshuju==customname
            print("选择正确")
        except Exception as e:
            print(e)
        #获取这个选择字段的类型
        webenleixing=driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)>span:nth-child(3)")
        try:
            assert wenbenshuju == "文本"
            print("类型正确")
        except Exception as e:
                print(e)
        #选取这个文本字段在查询器中展示
        driver.find_element_by_css_selector("#form-group_container1>div:nth-last-child(1)>input[type='checkbox']").click()
        driver.find_element_by_css_selector("[id='selectColumnSubmint']").click()
        #获取视图选择后的判断
        wenbenshuju1=driver.find_element_by_css_selector("tbody[class='info']>tr:last-child>td:first-child").text
        try:
            assert wenbenshuju1 == customname
            print("类型正确")
        except Exception as e:
                print(e)
        webenleixing1=driver.find_element_by_css_selector("tbody[class='info'']>tr:last-child>td:nth-child(2)").text
        try:
            assert webenleixing1 == "文本"
            print("类型正确")
        except Exception as e:
                print(e)
        driver.find_element_by_css_selector("[name='btnSubmit']").click()
        time.sleep(5)
        chaxunqiname=driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child(1)>td:nth-child(2)").text
        querierdescriptionname=driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child(1)>td:nth-child(3)").text
        try:
            assert chaxunqiname == queriername
            print("查询器标题保存没问题")
        except Exception as e:
                print(e)
        time.sleep(5)
        try:
            assert querierdescriptionname == querierdescription
            print("查询器描述保存没问题")
        except Exception as e:
                print(e)
        time.sleep(5)
        #获取获取查询器的个数  现在只获取数字  但是现在获取的是全部的信息
        querier = driver.find_element_by_css_selector("#page-wrapper>div>div:nth-child(2)>table>tbody>tr:nth-child(1)> td").text
        queriercount=querier[7:8]#这里得再调整一下
        print (queriercount)
        #获取活动的查询器的名称 进行遍历对比
        for index in range(int(queriercount.strip())):
            # print (index + 1)
            #这里得做一次处理取出错不包括空格的数据 但是现在获取的是有空格的信息
            querier_value = driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child('+ str(index+1)+')>td:nth-child(2)").text
            service_winid_value = querier_value.split('(')[1].split(')')[0]
            try:
                assert service_winid_value ==chaxunqiname
            except Exception as e:
                print(e)
                print("新建的查询器未出现在活动的列表中")

        # 获取失效的查询器的个数  只获取数字  但是现在获取的是全部的信息
        querier1 = driver.find_element_by_css_selector("#page-wrapper>div>div:nth-child(2)>table>tbody>tr:nth-child(1)> td").text
        queriercount1 = querier1[7:8]  # 这里得再调整一下
        # 获取非活动的查询器的名称 进行遍历对比
        for index in range(int(queriercount1.strip())):
                # print (index + 1)
                # 这里得做一次处理取出错不包括空格的数据 但是现在获取的是有空格的信息
            querier_value = driver.find_element_by_css_selector("div[class='col-lg-12 animated fadeInRight']>div:nth-child(3) tbody>tr:nth-child('+ str(index+1)+')>td:nth-child(2)").text
            service_winid_value = querier_value.split('(')[1].split(')')[0]
            try:
                assert service_winid_value != chaxunqiname
            except Exception as e:
                print(e)
                print("新建的查询器出现在非活动的列表中")

    def shixiao_querier(self, driver, customname, queriername, querierdescription):
        driver.find_element_by_css_selector


# imports
from couchbase.bucket import Bucket
from couchbase.n1ql import N1QLQuery
import pandas as pd

# helper function to flatten the json for data frame construction
def flatten_json(y):
    out = {}

    def flatten(x, name=''):
        if type(x) is dict:
            for a in x:
                flatten(x[a], name + a + '_')
        elif type(x) is list:
            i = 0
            for a in x:
                flatten(a, name + str(i) + '_')
                i += 1
        else:
            out[name[:-1]] = x

    flatten(y)
    return out

# get connection
bkt = Bucket('http://54.235.239.211:8091/default')

# run query
query = N1QLQuery("SELECT * FROM default WHERE ImageGuid IS NOT NULL")
rows = bkt.n1ql_query(query)

# create data frame for each result pulled from query
df_appended = None
for row in rows:
    result = row['default']
    # create data frames for items unique to result
    image_guid = flatten_json(result['ImageGuid'])
    guid_df = pd.io.json.json_normalize(image_guid)
    guid_df.columns = ['Guid']
    results_key = flatten_json(result['ResultsKey'])
    key_df = pd.io.json.json_normalize(results_key)
    key_df.columns = ['ResultsKey']
    # iterate over faces in result
    for i in range(len(result['EmotionResults'])):
        emotion_results = flatten_json(result['EmotionResults'][i])
        emotion_df = pd.io.json.json_normalize(emotion_results)
        face_results = flatten_json(result['FaceResults'][i])
        face_df = pd.io.json.json_normalize(face_results)
        face_df = face_df.drop(['FaceRectangle_Left', 'FaceRectangle_Top', 'FaceRectangle_Height', 'FaceRectangle_Width'], axis = 1)
        df_row = pd.concat([guid_df, key_df, emotion_df, face_df], axis = 1)
        # concatenate all 4 data frames
        if df_appended is None:
            df_appended = df_row
        else:
            df_appended = pd.concat([df_appended, df_row], axis = 0)


import sys
import torch
from torch import nn
from torch.nn import functional as F

# sys.path.append('../')
#
# from image.vqvae import Quantize

# Copyright 2018 The Sonnet Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or  implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

# Borrowed from https://github.com/deepmind/sonnet and ported it to PyTorch

class Quantize(nn.Module):
    def __init__(self, dim, n_embed, decay=0.99, eps=1e-5, qw=10):
        super().__init__()

        self.qw = qw
        self.dim = dim
        self.n_embed = n_embed
        self.decay = decay
        self.eps = eps

        embed = torch.randn(dim, n_embed)
        self.register_buffer('embed', embed)
        self.register_buffer('cluster_size', torch.zeros(n_embed))
        self.register_buffer('embed_avg', embed.clone())

    def forward(self, input):
        flatten = input.reshape(-1, self.dim)
        # print('flatten input {}'.format(flatten.shape))
        # print('quantize embed {}'.format(self.embed))
        dist = (
                flatten.pow(2).sum(1, keepdim=True)
                - 2 * flatten @ self.embed
                + self.embed.pow(2).sum(0, keepdim=True)
        )
        # print('quantize dist {}'.format(dist.shape))
        _, embed_ind = (-dist).max(1)
        # print('quantize embed_ind {}'.format(embed_ind.shape))
        embed_onehot = F.one_hot(embed_ind, self.n_embed).type(flatten.dtype)
        embed_ind = embed_ind.view(*input.shape[:-1])
        quantize = self.embed_code(embed_ind)

        if self.training:
            self.cluster_size.data.mul_(self.decay).add_(
                1 - self.decay, embed_onehot.sum(0)
            )
            embed_sum = flatten.transpose(0, 1) @ embed_onehot
            self.embed_avg.data.mul_(self.decay).add_(1 - self.decay, embed_sum)
            n = self.cluster_size.sum()
            cluster_size = (
                    (self.cluster_size + self.eps) / (n + self.n_embed * self.eps) * n
            )
            embed_normalized = self.embed_avg / cluster_size.unsqueeze(0)
            self.embed.data.copy_(embed_normalized)
        norm = input.detach().pow(2).mean() + self.qw * quantize.detach().pow(2).mean()
        # print(norm)
        diff = ((quantize.detach() - input).pow(2).mean()) / norm
        quantize = input + (quantize - input).detach()
        # print('quantize quantize {}'.format(quantize.shape))
        return quantize, diff, embed_ind

    def embed_code(self, embed_id):
        return F.embedding(embed_id, self.embed.transpose(0, 1))

class ResBlock(nn.Module):
    def __init__(self, in_channel, channel):
        super().__init__()

        self.conv = nn.Sequential(
            nn.ReLU(inplace=True),
            nn.Conv2d(in_channel, channel, 3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(channel, in_channel, 1),
        )

    def forward(self, input):
        out = self.conv(input)
        out += input

        return out

class Encoder(nn.Module):
    def __init__(self, in_channel, channel, n_res_block, n_res_channel, stride):
        super().__init__()
        if stride == 8:
            blocks = [
                nn.Conv2d(in_channel, channel // 2, 4, stride=2, padding=1),
                nn.ReLU(inplace=True),
                nn.Conv2d(channel // 2, channel, 4, stride=2, padding=1),
                nn.ReLU(inplace=True),
                nn.Conv2d(channel, channel, 3, padding=1),
                nn.Conv2d(in_channel, channel // 2, 4, stride=2, padding=1),
                nn.ReLU(inplace=True),
                nn.Conv2d(channel // 2, channel, 3, padding=1),
            ]

        elif stride == 4:
            blocks = [
                nn.Conv2d(in_channel, channel // 2, 4, stride=2, padding=1),
                nn.ReLU(inplace=True),
                nn.Conv2d(channel // 2, channel, 4, stride=2, padding=1),
                nn.ReLU(inplace=True),
                nn.Conv2d(channel, channel, 3, padding=1),
            ]

        elif stride == 2:
            blocks = [
                nn.Conv2d(in_channel, channel // 2, 4, stride=2, padding=1),
                nn.ReLU(inplace=True),
                nn.Conv2d(channel // 2, channel, 3, padding=1),
            ]

        for i in range(n_res_block):
            blocks.append(ResBlock(channel, n_res_channel))

        blocks.append(nn.ReLU(inplace=True))

        self.blocks = nn.Sequential(*blocks)

    def forward(self, input):
        return self.blocks(input)

class Decoder(nn.Module):
    def __init__(
            self, in_channel, out_channel, channel, n_res_block, n_res_channel, stride
    ):
        super().__init__()

        blocks = [nn.Conv2d(in_channel, channel, 3, padding=1)]

        for i in range(n_res_block):
            blocks.append(ResBlock(channel, n_res_channel))

        blocks.append(nn.ReLU(inplace=True))

        if stride == 8:
            blocks.extend(
                [
                    nn.ConvTranspose2d(channel, channel // 2, 4, stride=2, padding=1),
                    nn.ReLU(inplace=True),
                    nn.ConvTranspose2d(
                        channel // 2, out_channel, 4, stride=2, padding=1
                    ),
                    nn.ConvTranspose2d(channel, out_channel, 4, stride=2, padding=1),
                ]
            )

        elif stride == 4:
            blocks.extend(
                [
                    nn.ConvTranspose2d(channel, channel // 2, 4, stride=2, padding=1),
                    nn.ReLU(inplace=True),
                    nn.ConvTranspose2d(
                        channel // 2, out_channel, 4, stride=2, padding=1
                    ),
                ]
            )

        elif stride == 2:
            blocks.append(
                nn.ConvTranspose2d(channel, out_channel, 4, stride=2, padding=1)
            )

        self.blocks = nn.Sequential(*blocks)

    def forward(self, input):
        return self.blocks(input)

class VQVAE_ML(nn.Module):
    def __init__(
            self,
            in_channel=3,
            channel=64,
            n_res_block=2,
            n_res_channel=16,
            embed_dim=16,
            n_level=4,
            n_embed = 16,
            decay=0.80,
            stride=4,
    ):
        super().__init__()
        self.device = 'cuda'
        self.enc = Encoder(in_channel, channel, n_res_block, n_res_channel, stride=stride)
        self.enc_t = Encoder(channel, channel, n_res_block, n_res_channel, stride=2)
        self.quantize_conv = nn.Conv2d(channel, embed_dim, 1)
        self.n_level = n_level
        self.quantizes = nn.ModuleList()
        self.quantizes_conv = nn.ModuleList()
        self.bns = nn.ModuleList()
        for i in range(n_level):
            self.quantizes.append(Quantize(embed_dim, n_embed,decay))
            self.quantizes_conv.append(nn.Conv2d(embed_dim, embed_dim, 1))
            self.bns.append(nn.BatchNorm2d(embed_dim))
        self.dec_t = Decoder(
            embed_dim, embed_dim, channel, n_res_block, n_res_channel, stride=2
        )
        # self.quantize_conv_b = nn.Conv2d(embed_dim + channel, embed_dim, 1)
        # self.quantize_b = Quantize(embed_dim, n_embed)
        # self.upsample_t = nn.ConvTranspose2d(
        #     embed_dim, embed_dim, 4, stride=2, padding=1
        # )
        self.dec = Decoder(
            embed_dim ,
            in_channel,
            channel,
            n_res_block,
            n_res_channel,
            stride=stride,
        )

    def forward(self, input):
        quant, diff, _,_ = self.encode(input)
        # print('quant shape {}'.format(quant.shape))
        dec = self.decode(quant)

        return dec, diff

    def encode(self, input):
        enc = self.enc(input)
        enc = self.enc_t(enc)
        bottleneck = self.quantize_conv(enc)
        ids = None
        quants = None
        diffs = None
        quant_sum = None
        total_quantize = len(self.quantizes)
        for i,quantize in enumerate(self.quantizes):
            quant, diff, id = quantize(bottleneck.permute(0, 2, 3, 1))
            quant = quant.permute(0, 3, 1, 2)
            diff = diff.unsqueeze(0)

            if diffs is None:
                # diffs = total_quantize * diff
                diffs = diff
                quant_sum = quant
                quants = quant.unsqueeze(1)
                ids = id.unsqueeze(1)
            else:
                # diffs += (total_quantize - i) * diff
                diffs += diff
                quant_sum += quant
                quants = torch.cat((quants,quant.unsqueeze(1)),dim=1)
                ids = torch.cat((ids, id.unsqueeze(1)), dim=1)
            bottleneck -= quant
            # bottleneck = F.relu(self.bns[i](self.quantizes_conv[i](bottleneck)))
        return quant_sum, diffs, quants, ids

    def decode(self, quant):
        dec = self.dec_t(quant)
        return self.dec(dec)

    def decode_code(self, codes):
        quants = None
        for i, code in enumerate(codes):
            quant = self.quantizes.embed_code(code)
            quant = quant.permute(0, 3, 1, 2)
            quants += quant
        dec = self.decode(quants)

        return dec


#-*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup

url = "https://www.advokatsamfundet.se/Behover-du-advokat/Sok-advokat/Sokresultat/?positions=102001"
response = requests.get(url)
#print(response.text)
soup = BeautifulSoup(response.text)
divs = soup.find_all('div', {'class': "col-1"})
for div in divs:
	link = div.find("a")
	#print(link)
	if link:
		href = link["href"]
		sub_url = "https://www.advokatsamfundet.se" + href
		print(sub_url)

"""
print(sub_url)

	link = soup.find("div", {'class': "lead"})
	#else:
	#	description = ""
	print(mp_name.text)
	if link:
		description = link.find("p")
		print(description.text)
"""
"""

	link =
	url = "https://www.advokatsamfundet.se/" + link

def slugify(argument):
	slug = argument.lower()\
		.replace(" ", "-")\
		.replace("ö", "o")\
		.replace("ä", "a")\
		.replace("å", "a")\
		.replace("Å", "a")\
		.replace("é", "e")\
		.replace("Ö", "o")\
		.replace("ü", "u")
	return slug

for argument in miljopartiet.mps:
	changed_name = slugify(argument)
	url = "http://www.mp.se/om/" + changed_name
	# = response.text
	response = requests.get(url)
	soup = BeautifulSoup.BeautifulSoup(response.text)
#response.text innehåller html-koden
	mp_name = soup.find("h1")
	#mp_uppdrag = soup.find('div', {'class': "lead"})
	mp_email = soup.find('a', {'class': "email"})
	mp_mobile = soup.find('a', {'class': "tel"})
	div_tagg = soup.find("div", {'class': "lead"})
	#else:
	#	description = ""
	print(mp_name.text)
	if div_tagg:
		description = div_tagg.find("p")
		print(description.text)
	#print(mp_uppdrag.text)
	if mp_email:
		print(mp_email.text)
	if mp_mobile:
		print(mp_mobile.text)
	#print(slugify(argument))

	"""


__author__ = 'jim'

from scrapy.http import Request

from alascrapy.items import ProductItem, CategoryItem, ReviewItem
from alascrapy.spiders.base_spiders.ala_spider import AlaSpider
from alascrapy.lib.generic import get_full_url, date_format

class DartyComSpider(AlaSpider):
    name = 'darty_com'
    allowed_domains = ['darty.com']
    start_urls = ['http://darty.com/']

    def parse(self, response):
        sub_category_urls = self.extract_list(response.xpath(
                "//ul[@class='level-2']/li/a[contains(@href,'nav')]/@href"))

        for sub_category_url in sub_category_urls:
            yield Request(url=get_full_url(response, sub_category_url), callback=self.parse_sub_category)

    def parse_sub_category(self, response):
        category_urls = self.extract_list(response.xpath(
                '//li[@class="visible" or @class="supplementaire"]/a[contains(@href,"achat") or contains(@href,"cat=")]/@href'))

        for category_url in category_urls:
            yield Request(url=get_full_url(response, category_url), callback=self.parse_category)

    def parse_category(self, response):
        container_style = response.xpath("//div[contains(@class, 'product_detail')]")

        if container_style:
            category = CategoryItem()
            category['category_path'] = self.extract_all(
                    response.xpath('//div[@id="header-breadcrumb-zone"]//a/text()'), " > ")
            category['category_leaf'] = self.extract(response.xpath(
                    '//div[@id="header-breadcrumb-zone"]//li[last()]/a/text()'))
            category['category_url'] = response.url
            yield category

            if not self.should_skip_category(category):
                filtered_category_url = response.url+'?p=200&avis=100'
                request = Request(url=filtered_category_url, callback=self.parse_filtered_category)
                request.meta['ocn'] = category['category_path']
                yield request

    def parse_filtered_category(self, response):
        next_page_link = self.extract(response.xpath("//div[@id='main_pagination_bottom']//a[contains(text(), 'Page suivante')]/@href"))
        if next_page_link:
            next_page_link = get_full_url(response, next_page_link)
            request = Request(url=next_page_link, callback=self.parse_filtered_category)
            request.meta['ocn'] = response.meta['ocn']
            yield request

        product_urls = self.extract_list(response.xpath('//h2/a/@href'))
        for product_url in product_urls:
            product_url = get_full_url(response, product_url)
            request = Request(url=product_url, callback=self.parse_product)
            request.meta['ocn'] = response.meta['ocn']
            yield request

    def parse_product(self, response):
        product = ProductItem()

        product['TestUrl'] = response.url
        product['OriginalCategoryName'] = response.meta['ocn']
        product['ProductName'] = self.extract_all(response.xpath('//div[@itemprop="name" or contains(@class,"product_name")]/span//text()'))
        if not product['ProductName']:
            product['ProductName'] = self.extract_all(response.xpath('//div[@itemprop="name" or contains(@class,"product_name")]//text()'))
        product['PicURL'] = self.extract(response.xpath('//img[@itemprop="image"]/@src'))
        product['ProductManufacturer'] = self.extract(response.xpath('//a[@id="darty_product_brand"]/text()'))
        product['source_internal_id'] = self.extract(response.xpath('//@data-comparator-codic'))
        yield product

        reviews = response.xpath('//div[contains(@class,"bloc_reviews_item_full")]')

        for review in reviews:
            user_review = ReviewItem()
            user_review['DBaseCategoryName'] = "USER"
            user_review['ProductName'] = product['ProductName']
            user_review['TestUrl'] = product['TestUrl']
            user_review['source_internal_id'] = product['source_internal_id']
            date = self.extract(review.xpath('.//span/text()'))
            user_review['TestDateText'] = date_format(date.strip('Avis soumis le '), '%d/%m/%Y')
            user_review['SourceTestRating'] = self.extract(review.xpath('.//div[@class="bloc_reviews_note"]/text()[1]'))
            user_review['Author'] = self.extract(review.xpath('.//b/text()'))
            user_review['TestTitle'] = self.extract(review.xpath('.//h3/text()'))
            user_review['TestSummary'] = self.extract_all(review.xpath('.//p[contains(@class,"text")]//text()'))
            yield user_review


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
本示例代码仅作参考。如果您是业务调用，建议对body体也做加密。
注意：本示例用到的鉴权方式不适用于一部分产品（语音识别、人脸识别等）。请查阅您使用产品的API文档，如果鉴权方式基于Access Key（包括Access Key ID（AK）和Secret Access Key（SK）），
且最终认证字符串为bce-auth-v{version}/{accessKeyId}/{timestamp}/{expirationPeriodInSeconds}/{signedHeaders}/{signature}，则适用本示例。
此外，部分产品在AK/SK加密之外，还会要求额外的认证，例如百度信息流推广API需要您将百度推广账号的信息填到body体里。请注意查阅文档。

'''

import hashlib
import hmac
import urllib.parse
import time

# 1.AK/SK、host、method、URL绝对路径、querystring

AK = "c02376d42a05434c98*************"
SK = "08ead63a52f84ad****************"
host = "iotdm.gz.baidubce.com"
method = "GET"
query = ""
deviceName = "wu_t_shadow"
URI = "/v3/iot/management/device/"

# 2.x-bce-date
x_bce_date = time.gmtime()
x_bce_date = time.strftime('%Y-%m-%dT%H:%M:%SZ',x_bce_date)
#x_bce_date = "2020-03-20T13:33:41Z"
# 3.header和signedHeaders
header = {
        "Host":host,
        "Content-type":"application/json; charset=utf-8"
        }
signedHeaders = "content-type;host;x-bce-date"
# 4.认证字符串前缀
authStringPrefix = "bce-auth-v1" + "/" +AK + "/" + x_bce_date + "/" + "1800"
# 5.生成CanonicalRequest
#5.1生成CanonicalURI
CanonicalURI = urllib.parse.quote(URI)        # windows下为urllib.parse.quote，Linux下为urllib.quote
#5.2生成CanonicalQueryString
CanonicalQueryString = query           # 如果您调用的接口的query比较复杂的话，需要做额外处理
#5.3生成CanonicalHeaders
result = []
for key,value in header.items():
    tempStr = str(urllib.parse.quote(key.lower(),safe="")) + ":" + str(urllib.parse.quote(value,safe=""))
    result.append(tempStr)
result.sort()
CanonicalHeaders = "\n".join(result)
print("--------CanonicalHeaders---------" + CanonicalHeaders)
#5.4拼接得到CanonicalRequest
CanonicalRequest = method + "\n" + CanonicalURI + deviceName + "\n" + CanonicalQueryString +"\n" + "host:" + host
# 6.生成signingKey
signingKey = hmac.new(SK.encode('utf-8'),authStringPrefix.encode('utf-8'),hashlib.sha256)
print("---------signingKey------" + signingKey.hexdigest())
# 7.生成Signature
Signature = hmac.new((signingKey.hexdigest()).encode('utf-8'),CanonicalRequest.encode('utf-8'),hashlib.sha256)
print(signingKey.hexdigest().encode('utf-8'))
print(CanonicalRequest.encode('utf-8'))
print("---------Signature------" + Signature.hexdigest())

# 8.生成Authorization并放到header里
header['Authorization'] = authStringPrefix + "/host/" + Signature.hexdigest()
# 9.发送API请求并接受响应
print(header)
import requests
import json

body={
      "name" : "QQQQQQ"
      }

url = "http://"+ host + URI + deviceName
print(url)
r = requests.get(url,headers = header,data=json.dumps(body))

print(r.text)


import difflib
import nltk

def showdiff(fr, to):
    differ = difflib.Differ()
    html=""

    for entry in differ.compare(nltk.sent_tokenize(fr), nltk.sent_tokenize(to)):
        if entry[0]=="+":
            html+="<tt style='white-space: pre;background-color:lightgreen'>"+entry+"</tt>
"
        elif entry[0]=="-":
            html+="<tt style='white-space: pre;background-color:lightcoral'>"+entry+"</tt>
"
        else:
            html+="<tt style='white-space: pre;'>"+entry+"</tt>
"

    return html


#!/usr/bin/env python
#
# Copyright (c) 2015 Intel Corporation.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of works must retain the original copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the original copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of Intel Corporation nor the names of its contributors
#   may be used to endorse or promote products derived from this work without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors:
#         Hongjuan, Wang<hongjuanx.wang@intel.com>

import unittest
import os, sys, commands
import comm

class TestPackertoolsFunctions(unittest.TestCase):
  def test_permission_duplicate(self):
      targetDir = "testapp/permission_field_duplicate_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_lowercase(self):
      targetDir = "testapp/permission_field_lowercase_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_more2(self):
      targetDir = "testapp/permission_field_more2_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_uppercase(self):
      targetDir = "testapp/permission_field_uppercase_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_duplicate2(self):
      comm.setUp()
      cmd = "python %smake_apk.py --package=org.xwalk.permission --name=example --project-dir=permission --arch=%s --mode=%s --app-url=http://www.intel.com --permission=Contacts --permission=contacts --permission=CONTACTS" % (comm.Pck_Tools, comm.ARCH, comm.MODE)
      num = 2
      comm.perm2(num, cmd, self)

  def test_permission_value3(self):
      comm.setUp()
      cmd = "python %smake_apk.py --package=org.xwalk.permission --name=example --project-dir=permission --arch=%s --mode=%s --app-url=http://www.intel.com --permission=Geolocation --permission=Messaging --permission=Contacts" % (comm.Pck_Tools, comm.ARCH, comm.MODE)
      num = 3
      comm.perm2(num, cmd, self)

  def test_permission_value4(self):
      comm.setUp()
      cmd = "python %smake_apk.py --package=org.xwalk.permission --name=example --project-dir=permission --arch=%s --mode=%s --app-url=http://www.intel.com --permission=Contacts --permission=Geolocation --permission=Messaging --permission=Contacts:Geolocation:Messaging" % (comm.Pck_Tools, comm.ARCH, comm.MODE)
      num = 4
      comm.perm2(num, cmd, self)

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This module initializes the VPCs necessary for the rest of cloud formation

from itertools import chain

from troposphere import Parameter, Ref, Tags, Join, Output, Select, FindInMap
import troposphere.ec2 as ec2

import config as cfn
from config import CLOUDNAME

def emit_configuration():
    # Build the VPC here
    template = cfn.template

    # Parameters here
    nat_instance_class = template.add_parameter(
        Parameter(
            'NatInstanceType', Type='String', Default='t2.micro',
            Description='NAT instance type',
            AllowedValues=cfn.usable_instances(),
            ConstraintDescription='Instance size must be a valid instance type'
        )
    )

    keyname_param = template.add_parameter(
        Parameter(
            'KeyName', Type='AWS::EC2::KeyPair::KeyName',
            Description='Name of an existing EC2 KeyPair to enable SSH access into machines'
        )
    )

    cfn.keyname = keyname_param

    vpc = template.add_resource(
        ec2.VPC(
            'VPC',
            CidrBlock='{0}.0.0/16'.format(cfn.CIDR_PREFIX),
            EnableDnsSupport=True,
            EnableDnsHostnames=True,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, Ref('AWS::Region')]))
        )
    )

    # Add the VPC to cloudformation
    cfn.vpcs.append(vpc)

    gateway = template.add_resource(
        ec2.InternetGateway(
            'InternetGateway',
            Tags=Tags(Name='InternetGateway-{0}'.format(CLOUDNAME)),
            DependsOn=vpc.title
        )
    )

    gateway_attachment = template.add_resource(
        ec2.VPCGatewayAttachment(
            'GatewayAttachment',
            VpcId=Ref(vpc),
            InternetGatewayId=Ref(gateway),
            DependsOn=gateway.title
        )
    )

    public_routing_table = template.add_resource(
        ec2.RouteTable(
            '{0}PublicRouteTable'.format(cfn.VPC_NAME),
            VpcId=Ref(vpc),
            DependsOn=vpc.title
        )
    )

    # Add in the public route through the gateway
    template.add_resource(
        ec2.Route(
            'PublicRoute',
            RouteTableId=Ref(public_routing_table),
            DestinationCidrBlock=cfn.DEFAULT_ROUTE,
            GatewayId=Ref(gateway),
            DependsOn=gateway_attachment.title
        )
    )

    # Define the Security Group for the NATs
    nat_ingress_rules = [
        ec2.SecurityGroupRule(
            IpProtocol='tcp', CidrIp=cfn.DEFAULT_ROUTE, FromPort=p, ToPort=p
        ) for p in [22, 80, 443, 11371]
    ]

    nat_egress_rules = [
        ec2.SecurityGroupRule(
            IpProtocol='-1', CidrIp=cfn.DEFAULT_ROUTE, FromPort=0, ToPort=65535,
        )
    ]

    nat_security_group = template.add_resource(
        ec2.SecurityGroup(
            'NATSecurityGroup',
            GroupDescription='Security Group for NAT instances',
            VpcId=Ref(vpc),
            SecurityGroupIngress=nat_ingress_rules,
            SecurityGroupEgress=nat_egress_rules,
            DependsOn=vpc.title
        )
    )

    platform_subnets = list()
    master_subnets = list()
    public_subnets = list()
    vpn_subnets = list()
    worker_subnets = list()
    database_subnets = list()
    subnet_identifier = 'private'

    for idx, zone in enumerate(cfn.get_availability_zones()):
        region = Ref('AWS::Region')
        full_region_descriptor = Join('', [region, zone])

        # create a public subnet in each availability zone
        public_subnet = template.add_resource(
            ec2.Subnet(
                '{0}PublicSubnet{1}'.format(cfn.VPC_NAME, zone),
                VpcId=Ref(vpc),
                CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 80 + idx),
                AvailabilityZone=full_region_descriptor,
                DependsOn=vpc.title,
                Tags=Tags(Name=Join('-', [cfn.VPC_NAME, 'public-subnet', full_region_descriptor]))
            )
        )
        public_subnets.append(public_subnet)

        # Associate the public routing table with the subnet
        template.add_resource(
            ec2.SubnetRouteTableAssociation(
                '{0}PublicRouteTableAssociation'.format(public_subnet.title),
                SubnetId=Ref(public_subnet),
                RouteTableId=Ref(public_routing_table),
                DependsOn=public_subnet.title
            )
        )

        # Create the NAT instance in the public subnet
        nat_name = '{0}Nat{1}'.format(cfn.VPC_NAME, zone)
        nat_instance = template.add_resource(
            ec2.Instance(
                nat_name,
                DependsOn=vpc.title,
                InstanceType=Ref(nat_instance_class),
                KeyName=Ref(keyname_param),
                SourceDestCheck=False,
                ImageId=FindInMap('RegionMap', region, int(cfn.Amis.NAT)),
                NetworkInterfaces=[
                    ec2.NetworkInterfaceProperty(
                        Description='Network interface for {0}'.format(nat_name),
                        GroupSet=[Ref(nat_security_group)],
                        SubnetId=Ref(public_subnet),
                        AssociatePublicIpAddress=True,
                        DeviceIndex=0,
                        DeleteOnTermination=True
                    )
                ],
                Tags=Tags(Name=Join('-', [cfn.VPC_NAME, 'nat', full_region_descriptor]))
            )
        )

        # Associate the private routing table with the NAT
        private_routing_table = template.add_resource(
            ec2.RouteTable(
                '{0}PrivateRouteTable{1}'.format(cfn.VPC_NAME, zone),
                VpcId=Ref(vpc),
                DependsOn=nat_instance.title,
                Tags=Tags(Name=Join('-', [cfn.VPC_NAME, 'private-route-table', full_region_descriptor]))
            )
        )

        template.add_resource(
            ec2.Route(
                'PrivateRoute{0}'.format(zone.upper()),
                RouteTableId=Ref(private_routing_table),
                DestinationCidrBlock=cfn.DEFAULT_ROUTE,
                InstanceId=Ref(nat_instance),
                DependsOn=private_routing_table.title
            )
        )

        worker_subnet = ec2.Subnet(
            '{0}{1}WorkerSubnet{2}'.format(cfn.VPC_NAME, subnet_identifier, zone.upper()),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/22'.format(cfn.CIDR_PREFIX, 100 + (idx * 4)),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, '{0}-worker-subnet'.format(subnet_identifier), full_region_descriptor]))
        )
        worker_subnets.append(worker_subnet)

        platform_subnet = ec2.Subnet(
            '{0}{1}PlatformSubnet{2}'.format(cfn.VPC_NAME, subnet_identifier, zone.upper()),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 10 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, '{0}-platform-subnet'.format(subnet_identifier), full_region_descriptor]))
        )
        platform_subnets.append(platform_subnet)

        master_subnet = ec2.Subnet('{0}{1}MasterSubnet{2}'.format(cfn.VPC_NAME, subnet_identifier, zone.upper()),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 90 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, '{0}-master-subnet'.format(subnet_identifier), full_region_descriptor]))
        )
        master_subnets.append(master_subnet)

        vpn_subnet = ec2.Subnet(
            '{0}VpnSubnet{1}'.format(cfn.VPC_NAME, zone),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 60 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('.', [full_region_descriptor, cfn.CLOUDNAME, cfn.CLOUDENV, "vpn-client"]))
        )
        vpn_subnets.append(vpn_subnet)

        database_subnet = ec2.Subnet(
            '{0}DatabaseSubnet{1}'.format(cfn.VPC_NAME, zone),
            VpcId=Ref(vpc),
            CidrBlock='{0}.{1}.0/24'.format(cfn.CIDR_PREFIX, 20 + idx),
            AvailabilityZone=full_region_descriptor,
            DependsOn=vpc.title,
            Tags=Tags(Name=Join('-', [cfn.VPC_NAME, subnet_identifier, 'database-subnet', full_region_descriptor]))
        )
        database_subnets.append(database_subnet)

    # Associate a routing table with each of the master/platform/worker subnets
    if cfn.USE_PRIVATE_SUBNETS:
        routing_table = private_routing_table
    else:
        routing_table = public_routing_table

    for sn in chain(worker_subnets, master_subnets, platform_subnets, database_subnets):
        template.add_resource(sn)
        template.add_resource(
            ec2.SubnetRouteTableAssociation(
                '{0}{1}RouteTableAssociation'.format(sn.title, subnet_identifier),
                SubnetId=Ref(sn),
                RouteTableId=Ref(routing_table),
                DependsOn=sn.title
            )
        )

    # associate the public routing table with the VPN subnets
    for sn in vpn_subnets:
        template.add_resource(sn),
        template.add_resource(
            ec2.SubnetRouteTableAssociation(
                '{0}RouteTableAssociation'.format(sn.title),
                SubnetId=Ref(sn),
                RouteTableId=Ref(public_routing_table),
                DependsOn=sn.title
            )
        )

    # affect the global state
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.PLATFORM, platform_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.PUBLIC, public_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.MASTER, master_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.WORKER, worker_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.VPN, vpn_subnets)
    cfn.add_vpc_subnets(vpc, cfn.SubnetTypes.DATABASE, database_subnets)


import pandas as pd
from bs4 import BeautifulSoup
import requests
import zipfile
import io
import re
import numpy as np

pd.set_option('float_format', '{:.6f}'.format)

survey = "https://drive.google.com/uc?export=download&id=0B6ZlG_Eygdj-c1kzcmUxN05VUXM"
data = "survey_results_public.csv"
schema = "survey_results_schema.csv"
populations = "https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population"

survey = requests.get(survey)
#download zip in memory
zp = zipfile.ZipFile(io.BytesIO(survey.content))

data = pd.read_csv(zp.open(data), index_col = 0)
schema = pd.read_csv(zp.open(schema))
#1

print(data.iloc[:, :1].head())
print(schema.head())
#I used index_col only to match output in assignment, it's uncomfortable for me to use
#therefore, reseting it
data = data.reset_index()
respondents = data["Respondent"].size
questions = schema["Question"].size
print("Number of respondents:", respondents)
print("Number of questions:", questions)
print()

#2
countries = (data[["Respondent", "Country"]]
       .groupby("Country").size()
       .sort_values(ascending = False))
print(countries.head(10))
print()
print(countries.head(10)/respondents)
print()

#3
populations = requests.get(populations)
soup = BeautifulSoup(populations.text, "html.parser")
populations  = soup.find('table', attrs = {'class' : 'wikitable sortable'})

table = []
headers = []

rows = populations.find_all('tr')
for th in populations.find_all('th'):
    headers.append(th.getText().strip().replace('\n',''))
for tr in populations.find_all('tr'):
    row = []
    for td in tr.find_all('td'):
        #deleting text between square brackets, on wiki there are notes for some countries in []
        td = re.sub(r'\[.*?\]', '', td.text)
        row.append(' '.join(td.split()))
    table.append(row)

populations = pd.DataFrame(table, columns = headers)
#from wiki downloaded table leaving only country and population
populations = populations.iloc[1:, 1:3]
populations.columns = ['Country', 'Population']
#change population from string to int
populations["Population"] = populations["Population"].str.replace(',', '').apply(int)
countries100 = countries[countries > 100]
#getting numeric index
countries100 = countries100.reset_index()
countries100.columns = ['Country', 'Developers']
#merging 2 datasets on 'Country' column
merged = pd.merge(countries100, populations, on = 'Country')
merged['Ratio'] = merged['Developers'].apply(int)/merged['Population']
#Adjust data to similar output in home assignment pdf
merged = merged.set_index('Country')
merged.columns = ['Country' , 'Population', 'Ratio']
del merged.index.name
print(merged.sort_values('Ratio', ascending = False).head(10))
print()
#4
versionControl = (data[["Respondent", "VersionControl"]]
       .groupby("VersionControl").size()
       .sort_values(ascending = False))
print(versionControl)
print()
#5
#removing white spaces in language string
dummyLanguages = data["HaveWorkedLanguage"].str.replace(r'\s', '')
#splitting language string into columns, with corresponding binary variables
dummyLanguages = dummyLanguages.str.get_dummies(sep = ';')
#all languages can be retrivied as array of columns after splitting
languages = list(dummyLanguages.columns)
print(languages)
print()
#6
print(dummyLanguages[dummyLanguages == 1]
      .count()
      .sort_values(ascending = False)
      .head(10))
print()
#7
#Counting how many users for each language in countries
dummyLanguages["Country"] = data["Country"]
countryPopular = (dummyLanguages
                  .groupby("Country")
                  .sum())
#Finding maximum value for language in row
countryPopular["Total"] = countryPopular.max(axis = 1)
#and corresponding column for that value
countryPopular["Popular"] = countryPopular.idxmax(axis = 1)
print(countryPopular[["Total", "Popular"]]
      .sort_values("Total", ascending = False)
      .head(10))
print()
print(countryPopular[["Total", "Popular"]]
      .loc[countryPopular["Popular"] != "JavaScript"]
      .sort_values("Total", ascending = False)
      .head(5))
print()
#let's find countries with highest copy paste rate, and countries which provide
#above average salary to copy pasters
cp = data[["Country", "StackOverflowCopiedCode", "Salary"]]
#get countries with at least 100 people in survey for more meaningful results
cp = cp[cp["Country"].isin(list(countries100["Country"]))]
#get list of unique answers on question about code copying, and select where person
#copied code at least once a day or once a week
indicators = list(cp.groupby("StackOverflowCopiedCode").max().index)[:2]
cp = cp[cp["StackOverflowCopiedCode"].isin(indicators)]
#group by country and calculating how many people copied code once a day or week, and their
#average salaries
cp = (cp.groupby('Country')
      .agg({'StackOverflowCopiedCode' : np.size, 'Salary' : np.mean}))
#reseting indices for proper division
cp = cp.reset_index()
countries100 = countries100.sort_values("Country").reset_index(drop = True)
cp["copypaste rate"] = (cp["StackOverflowCopiedCode"]/
                                countries100["Developers"])
#shows countries with highest level of copy paste in their code
print(cp.sort_values("copypaste rate", ascending = False).head(10))
print()
#countries where copypasters can earn salary higher than average in survey
print(cp[cp["Salary"] >= data["Salary"].mean()].head(10))
print()
#in which countries copy pasters earn salary higher than country average?
countryAvg = (data[['Country', 'Salary']]
              .groupby('Country', as_index = False)
              .mean())
countryAvg.columns = ['Country', 'Average Salary']
cp = pd.merge(cp, countryAvg, on = 'Country')
print(cp[cp["Salary"] > cp["Average Salary"]]
      [["Country","Salary","Average Salary"]]
      .head(10))


#import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns; sns.set(style="ticks", color_codes=True)

#GRÁFICA
data = pd.read_csv('datos.csv', header = 0)
data

plt.scatter(data['distancia'], data['tiempo_real'], c='r', label = 'Tiempo Real')
plt.scatter(data['distancia'], data['tiempo_medido'], c='b', label = 'Tiempo Medido')

plt.xlabel('distancia [m]')
plt.ylabel('tiempo [s]')
plt.title('Caida Libre')
plt.legend()
plt.grid(True)
plt.show()

#TABLAS DE ERRORES
media = data.tiempo_real.mean()
media

data['error_absoluto'] = abs(data.tiempo_real - data.tiempo_medido)
data.head()

data['error_relativo'] = data['error_absoluto'] / data.tiempo_real * 100
data.head()

#Gráficas
sns.pairplot(data, hue="distancia")


"""
Common DBus utilities.
"""

from __future__ import absolute_import

from gi.repository import Gio
from gi.repository import GLib

__all__ = ['DBusProperties',
           'DBusProxy',
           'DBusObject',
           'DBusBus',
           'DBusService',
           'DBusException']

DBusException = GLib.GError

class DBusProperties(object):

    """
    Dbus property map abstraction.

    Wraps properties of a DBus interface on a DBus object as attributes.

    :ivar Gio.DBusProxy __proxy: proxy for the DBus.Properties interface
    :ivar str __interface: inspected interface name
    """

    def __init__(self, object, name):
        """
        Initialize a proxy object with standard DBus property interface.

        :param Gio.DBusObject object: accessed object
        :param str name: interface name
        """
        self.__proxy = object._get_interface(
            'org.freedesktop.DBus.Properties')
        self.__interface = name

    def __getattr__(self, key):
        """
        Retrieve the property via the DBus proxy.

        :param str key: name of the dbus property
        :returns: the property value
        """
        return self.__proxy.Get('(ss)', self.__interface, key)

class DBusProxy(object):

    """
    DBus proxy object for a specific interface.

    Provides attribute accessors to properties and methods of a DBus
    interface on a DBus object.

    :ivar str object_path: object path of the DBus object
    :ivar DBusProperties property: attribute access to DBus properties
    :ivar Gio.DBusProxy method: attribute access to DBus methods
    :ivar Gio.DBusProxy _proxy: underlying proxy object
    """

    Exception = DBusException

    def __init__(self, proxy):
        """
        Initialize property and method attribute accessors for the interface.

        :param Gio.DBusProxy proxy: accessed object
        :param str interface: accessed interface
        """
        self._proxy = proxy
        self.object_path = proxy.get_object_path()
        self.property = DBusProperties(self.object,
                                       proxy.get_interface_name())
        self.method = proxy

    @property
    def object(self):
        """
        Get a proxy for the underlying object.

        :rtype: DBusObject
        """
        proxy = self._proxy
        return DBusObject(proxy.get_connection(),
                          proxy.get_name(),
                          proxy.get_object_path())

    def connect(self, event, handler):
        """
        Connect to a DBus signal.

        :param str event: event name
        :param handler: callback
        :returns: subscription id
        :rtype: int
        """
        interface = self._proxy.get_interface_name()
        return self.object.connect(interface, event, handler)

class DBusObject(object):

    """
    Simple proxy class for a DBus object.

    :param Gio.DBusConnection connection:
    :param str bus_name:
    :param str object_path:
    """

    def __init__(self, connection, bus_name, object_path):
        """
        Initialize member variables.

        :ivar Gio.DBusConnection connection:
        :ivar str bus_name:
        :ivar str object_path:

        This performs IO at all.
        """
        self.connection = connection
        self.bus_name = bus_name
        self.object_path = object_path

    def _get_interface(self, name):
        """
        Get a Gio native interface proxy for this Dbus object.

        :param str name: interface name
        :returns: a proxy object for the other interface
        :rtype: Gio.DBusProxy
        """
        return Gio.DBusProxy.new_sync(
            self.connection,
            Gio.DBusProxyFlags.NONE,
            info=None,
            name=self.bus_name,
            object_path=self.object_path,
            interface_name=name,
            cancellable=None,
        )

    def get_interface(self, name):
        """
        Get an interface proxy for this Dbus object.

        :param str name: interface name
        :returns: a proxy object for the other interface
        :rtype: DBusProxy
        """
        return DBusProxy(self._get_interface(name))

    @property
    def bus(self):
        """
        Get a proxy object for the underlying bus.

        :rtype: DBusBus
        """
        return DBusBus(self.connection, self.bus_name)

    def connect(self, interface, event, handler):
        """
        Connect to a DBus signal.

        :param str interface: interface name
        :param str event: event name
        :param handler: callback
        :returns: subscription id
        :rtype: int
        """
        object_path = self.object_path
        return self.bus.connect(interface, event, object_path, handler)

class DBusCallback(object):

    def __init__(self, handler):
        """Store reference to handler."""
        self._handler = handler

    def __call__(self,
                 connection,
                 sender_name,
                 object_path,
                 interface_name,
                 signal_name,
                 parameters,
                 *user_data):
        """Call handler unpacked signal parameters."""
        return self._handler(*parameters.unpack())

class DBusCallbackWithObjectPath(object):

    def __init__(self, handler):
        """Store reference to handler."""
        self._handler = handler

    def __call__(self,
                 connection,
                 sender_name,
                 object_path,
                 interface_name,
                 signal_name,
                 parameters,
                 *user_data):
        """Call handler with object_path and unpacked signal parameters."""
        return self._handler(object_path, *parameters.unpack())

class DBusBus(object):

    """
    Simple proxy class for a connected bus.

    :ivar Gio.DBusConnection connection:
    :ivar str bus_name:
    """

    def __init__(self, connection, bus_name):
        """
        Initialize member variables.

        :param Gio.DBusConnection connection:
        :param str bus_name:

        This performs IO at all.
        """
        self.connection = connection
        self.bus_name = bus_name

    def get_object(self, object_path):
        """
        Get a object representing the specified object.

        :param str object_path: object path
        :returns: a simple representative for the object
        :rtype: DBusObject
        """
        return DBusObject(self.connection, self.bus_name, object_path)

    def connect(self, interface, event, object_path, handler):
        """
        Connect to a DBus signal.

        :param str interface: interface name
        :param str event: event name
        :param str object_path: object path or ``None``
        :param handler: callback
        """
        if object_path:
            callback = DBusCallback(handler)
        else:
            callback = DBusCallbackWithObjectPath(handler)
        return self.connection.signal_subscribe(
            self.bus_name,
            interface,
            event,
            object_path,
            None,
            Gio.DBusSignalFlags.NONE,
            callback,
            None,
        )

    def disconnect(self, subscription_id):
        """
        Disconnect a DBus signal subscription.
        """
        self.connection.signal_unsubscribe(subscription_id)

class DBusService(object):

    """
    Abstract base class for UDisksX service wrapper classes.
    """

    @classmethod
    def connect_service(cls):
        """
        Connect to the service object on DBus.

        :returns: new proxy object for the service
        :rtype: DBusProxy
        :raises BusException: if unable to connect to service.
        """
        return DBusProxy(Gio.DBusProxy.new_for_bus_sync(
            Gio.BusType.SYSTEM,
            Gio.DBusProxyFlags.NONE,
            info=None,
            name=cls.BusName,
            object_path=cls.ObjectPath,
            interface_name=cls.Interface,
            cancellable=None,
        ))


"""
Problem 35

The number, 197, is called a circular prime because all rotations of the digits: 197, 971, and 719, are themselves prime.

There are thirteen such primes below 100: 2, 3, 5, 7, 11, 13, 17, 31, 37, 71, 73, 79, and 97.

How many circular primes are there below one million?

"""

from __future__ import print_function
from __future__ import division

from itertools import count, imap
from sympy import isprime
from math import log10

found = {}

def primes():
    for i in count(2):
        if isprime(i):
            yield(i)

def rotations(n):
    res = set()
    digit_count = int(log10(n))
    for i in range(1, digit_count+1):
        a, b = 10**i, 10**(digit_count-i+1)
        res.add(n%a * b + n//a)
    return res

c = 0
for i in primes():
    if i > 1000000:
        break
    if all(imap(isprime, rotations(i))):
        c += 1

print(c)


import pycqed as pq
import matplotlib.pyplot as plt
import os
from pycqed.analysis import measurement_analysis as ma
from numpy.testing import assert_almost_equal

class TestSSRODiscriminationAnalysis:

    @classmethod
    def setup_class(cls):
        cls.datadir = os.path.join(pq.__path__[0], 'tests', 'test_data')
        ma.a_tools.datadir = cls.datadir

    def test_butterfly_postselected(self):
        # Test the correct file is loaded
        a = ma.butterfly_analysis(timestamp='20170710_180002',
                                  close_main_fig=False, initialize=True,
                                  threshold=0.5,
                                  digitize=False, case=True)

        assert_almost_equal(a.butterfly_coeffs['F_a_butterfly'], 0.7998,
                            decimal=3)

    def test_butterfly_simple(self):
        # Test the correct file is loaded
        a = ma.butterfly_analysis(timestamp='20170710_182949',
                                  close_main_fig=False, initialize=False,
                                  threshold=0.5,
                                  digitize=False, case=True)

        assert_almost_equal(a.butterfly_coeffs['F_a_butterfly'], 0.819,
                            decimal=3)

    @classmethod
    def teardown_class(cls):
        plt.close('all')


import pymunk as pm
from random import randrange
import math
import constants

class RoadBuilder:
    def __init__(self, space):
        self._space = space

    def build_road(self, vs, radius, color=(0, 0, 0, 0)):
        """
        :param color: color of the road
        :param vs: Vertices that make up the roads
        :param radius: radius of the road Segments
        :return: The road Segment objects
        """
        # create the road Segments using the given vertices
        static_body = pm.Body(body_type=pm.Body.STATIC)
        static_segments = []
        for v in vs:
            seg = pm.Segment(static_body, v[0], v[1], radius)
            seg.elasticity = 0.10
            seg.friction = 0.90
            seg.color = color
            seg.filter = pm.ShapeFilter(categories=3)
            static_segments.append(seg)

        self._space.add(static_body)
        self._space.add(*static_segments)
        return static_body, static_segments

    def random_terrain_vertices_generator(self, vs, length, total_segments):
        """
        Randomly generates a road that builds towards the positive x-direction with
        each segment being a fixed length
        :param total_segments: total number of segments to be made
        :param length: length of the road Segment
        :param vs: Starting vertex
        :return: The vertices of the random terrain
        """
        vertices = [vs]
        # starting from 1 to be able to append new vertices to the vertices list that already contains 1
        for seg_num in range(1, total_segments*2):
            # most steep : 1, very flat: > 2
            steepness = 2
            # x-value has to be the same every other point to make the segments connect
            if seg_num % 2 == 1:
                delta_y = randrange(int(-length/steepness), int(length/steepness))
                delta_x = math.sqrt(length ** 2 - delta_y ** 2)
            else:
                delta_x = 0
                delta_y = 0
            # append a new vertex using the coordinates of the previous vertex
            prev_vertex = vertices[seg_num - 1]
            # make sure the delta_y value never create a point below the screen
            if prev_vertex[1] + int(delta_y) > constants.HEIGHT:
                delta_y = constants.HEIGHT - prev_vertex[1]
            # create the new vertex
            new_vertex = prev_vertex[0] + int(delta_x), prev_vertex[1] + int(delta_y)
            vertices.append(new_vertex)
        # put the vertices into pairs
        vertex_pairs = []
        for i in range(0, total_segments*2, 2):
            vertex_pairs.append((vertices[i], vertices[i+1]))
        return vertex_pairs