# coding: utf-8
# app: paquetes
# module: forms
# date: 25 Aug 2018
# author: Javier Sanchez Toledano <js.toledano@me.com>
# description: Formularios para distribución de paquetes

import datetime as dt
from django import forms
from apps.paquetes.models import Envio, EnvioModulo

class EnvioModuloForm(forms.ModelForm):
    class Meta:
        model = EnvioModulo
        fields = '__all__'

    mac = forms.CharField(
        widget=forms.TextInput(
            attrs={'class': 'form-control'}
        )
    )
    paquetes = forms.CharField(
        label='Número de paquetes',
        initial=1,
        widget=forms.TextInput(
            attrs={
                'class': 'form-control',
                'type': 'number'
            }
        )
    )
    formatos = forms.CharField(
        label='Número de FCPVF',
        widget=forms.TextInput(
            attrs={
                'class': 'form-control',
                'type': 'number'
            }
        )
    )
    recibido_mac = forms.DateTimeField(
        widget=forms.TextInput(
            attrs={
                'class': 'form-control',
                'placeholder': 'dd/mm/aaaa hh:mm'
            }
        )
    )
    disponible_mac = forms.DateTimeField(
        widget=forms.TextInput(
            attrs={
                'class': 'form-control',
                'placeholder': 'dd/mm/aaaa hh:mm'
            }
        )
    )

    def clean_disponible_mac(self):
        data = self.cleaned_data
        menor = data['recibido_mac']
        mayor = data['disponible_mac']
        envio = data['lote']
        corte = envio.recibido_vrd
        if menor > mayor:
            raise forms.ValidationError('La fecha de recepción debe ser anterior a la disponibilidad')
        if menor < corte:
            raise forms.ValidationError('La fecha de recepción en MAC es anterior a la de recepción en VRD %s' % corte)
        if mayor < corte:
            raise forms.ValidationError(
                'La fecha de disponibilidad en MAC es anterior a la de recepción en  %s' % corte
            )
        return data['disponible_mac']

class PrepareForm(forms.ModelForm):
    class Meta:
        model = Envio
        fields = '__all__'

    DISTRITO = (
        ('#', '-- Seleccionar --'),
        ('1', '1: Apizaco - Distrito 01'),
        ('2', '2: Tlaxcala - Distrito 02'),
        ('3', '3: Zacatelco - Distrito 03')
    )
    distrito = forms.CharField(
        widget=forms.Select(
            choices=DISTRITO,
            attrs={
                'class': 'form-control',
                'onchange': "dynamic_Select(\'/paquetes/distrito/\', this.value)"
            }
        )
    )
    modulos = forms.CharField(
        widget=forms.TextInput(
            attrs={'class': 'form-control', 'type': 'number'}
        )
    )

    lote = forms.CharField(
        widget=forms.TextInput(
            attrs={'class': 'input-small form-control'}
        ),
        initial='18_29_'
    )
    num_prod = forms.CharField(
        label='Número de producción',
        widget=forms.TextInput(
            attrs={'class': 'input-small form-control'}
        ),
    )
    tipo_lote = forms.ChoiceField(
        widget=forms.RadioSelect(attrs={
            'class': 'radio-inline'
        }),
        label="Tipo de Lote", choices=(
            ('ORD', 'ORD'),
            ('BIS', 'BIS'),
            ('REI', 'REI'),
            ('RA', 'RA'),
            ('EXT', 'EXT'),
            ),
        initial='ORD',
        help_text=u'Si aparece algún nuevo tipo de lote, comunícate con el Equipo Técnico SGC'
        )
    tipo_cinta = forms.ChoiceField(
        widget=forms.RadioSelect(attrs={
            'class': 'radio-inline'
        }),
        label='Tipo de Cinta',
        choices=(
            ('1', 'Actualizacion'),
            ('2', 'Recurso de Apelación'),
            ),
        initial='1',
        help_text='Si aparece algún nuevo tipo de cinta, comunícate con el Equipo Técnico SGC'
        )
    mac = forms.MultipleChoiceField(
        choices=EnvioModulo.MODULO,
        widget=forms.CheckboxSelectMultiple(
            attrs={'inline': True}
        )
    )
    credenciales = forms.CharField(
        widget=forms.TextInput(
            attrs={'class': 'form-control', 'type': 'number'}
        )
    )
    cajas = forms.CharField(
        widget=forms.TextInput(
            attrs={'class': 'form-control', 'type': 'number'}
        )
    )
    envio_cnd = forms.DateTimeField(
        label='Fecha y Hora del envío desde CND',
        widget=forms.TextInput(
            attrs={
                'class': 'form-control',
                'data-date-format': "DD/MM/YYYY HH:mm",
                'placeholder': 'dd/mm/aaaa hh:mm'
            }
        )
    )
    recibido_vrd = forms.DateTimeField(
        widget=forms.TextInput(
            attrs={
                'class': 'form-control',
                'data-date-format': "DD/MM/YYYY HH:mm",
                'placeholder': 'dd/mm/aaaa hh:mm'
            }
        )
    )

    def clean_recibido_vrd(self):
        recibido = self.cleaned_data['recibido_vrd']
        send = self.cleaned_data['envio_cnd']
        if send.date() >= recibido.date():
            raise forms.ValidationError('La fecha de Envío de CND es posterior a la recepción en  VRD')
        if recibido.date() >= dt.date.today():
            raise forms.ValidationError('La fecha de recepción es muy reciente')
        return recibido


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 29 15:04:19 2019

@author: jonas
"""

"""
1. Clear all variables
"""

def clear_all():
    """Clears all the variables from the workspace of the spyder application."""
    gl = globals().copy()
    for var in gl:
        if var[0] == '_': continue
        if 'func' in str(globals()[var]): continue
        if 'module' in str(globals()[var]): continue

        del globals()[var]
if __name__ == "__main__":
    clear_all()

"""
2. Import packages
"""

from collections import OrderedDict
import sys
import numpy as np
import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from spacy.lang.en.examples import sentences

import pandas as pd
import numpy as np
import nltk
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, PorterStemmer
from string import punctuation
from collections import Counter
import mpld3

from collections import OrderedDict
import re
import warnings
import itertools as it

warnings.filterwarnings("ignore", category=DeprecationWarning)

from html.parser import HTMLParser
from bs4 import BeautifulSoup

from matplotlib import pyplot as plt
from scipy.cluster.hierarchy import dendrogram, linkage
from scipy.cluster.hierarchy import cophenet
from scipy.spatial.distance import pdist

porter = PorterStemmer()
wnl = WordNetLemmatizer()
stop = stopwords.words('english')
stop.append("new")
stop.append("like")
stop.append("u")
stop.append("it'")
stop.append("'s")
stop.append("n't")
stop.append('mr.')
stop = set(stop)

porter = PorterStemmer()
wnl = WordNetLemmatizer()
stop = stopwords.words('english')
stop.append("new")
stop.append("like")
stop.append("u")
stop.append("it'")
stop.append("'s")
stop.append("n't")
stop.append('mr.')
stop = set(stop)

"""
Import Data
"""
#First, we import the three schemes respectively:
#import first sheet: SDG Goals
sdg = pd.read_excel('sdg.xlsx', header=None, encoding='latin', skip_blank_lines=False)
sdg.fillna(0, inplace=True)
sdg = sdg.where(sdg != "x", 1)
sdg = sdg.where(sdg != "X", 1)

#import second sheet: SDG Targets
target = pd.read_excel('Target.xlsx', header=None, encoding='latin', skip_blank_lines=False)
target.fillna(0, inplace=True)
target = target.where(target != "x", 1)
target = target.where(target != "X", 1)

#import third sheet: SDG Indicators
indicator = pd.read_excel('Indicator.xlsx', header=None, encoding='latin', skip_blank_lines=False)
indicator.fillna(0, inplace=True)
indicator = indicator.where(indicator != "x", 1)
indicator = indicator.where(indicator != "X", 1)

keys = sdg.iloc[0,2:1317] #could have used also indicators or targets to obtain keys
keys = keys.astype(str)
keys = keys.reset_index()
keys = keys.drop(['index'], axis=1)
#decapitalize keywords for better matching
a = keys[0].tolist()
a = [element.lower() for element in a]
strip = pd.DataFrame({0:a})

#because indicator has one key word more we use also the indicator keywords.
keys2 = indicator.iloc[0,4:1319]
keys2 = keys2.astype(str)
keys2 = keys2.reset_index()
keys2 = keys2.drop(['index'], axis=1)
a = keys[0].tolist()
a = [element.lower() for element in a]
strip2 = pd.DataFrame({0:a})

"""
Goals
"""
sdg.columns = sdg.iloc[0]
goals = sdg.iloc[1:18,0:2]
goals = goals.astype(str)
goals = goals.reset_index()
goals = goals.drop(['index'], axis=1)

#decapitalize goals for better matching
a = goals['Description'].tolist()
a = [element.lower() for element in a]
goals_lower = pd.DataFrame({'Description':a})
del goals['Description']
frames = [goals, goals_lower]
goals = pd.concat(frames, axis=1)

"""
Targets
"""
target.columns = target.iloc[0]
targets = target.iloc[1:170,0:3]
targets = targets.astype(str)
targets = targets.reset_index()
targets = targets.drop(['index'], axis=1)

#decapitalize targets for better matching
a = targets['Description'].tolist()
a = [element.lower() for element in a]
target_lower = pd.DataFrame({'Description':a})
del targets['Description']
frames = [targets, target_lower]
targets = pd.concat(frames, axis=1)

"""
Indicators
"""
indicator.columns = indicator.iloc[0]
indicators = indicator.iloc[1:246,0:4]
indicators = indicators.astype(str)
indicators = indicators.reset_index()
indicators = indicators.drop(['index'], axis=1)

#decapitalize indicators for better matching
a = indicators['Description'].tolist()
a = [element.lower() for element in a]
indicator_lower = pd.DataFrame({'Description':a})
del indicators['Description']
frames = [indicators, indicator_lower]
indicators = pd.concat(frames, axis=1)

"""
Goals + Targets + Indicators
"""
#group by goals and sum up string in targets and indicators
target_sum = targets.groupby(['Goal'],as_index=False).agg(lambda x : x.sum() if x.dtype=='float64' else ' '.join(x))
indicator_sum = indicators.groupby(['Goal'],as_index=False).agg(lambda x : x.sum() if x.dtype=='float64' else ' '.join(x))

goals['target'] = target_sum['Description']
goals['indicator'] = indicator_sum['Description']
#rename SDG
goals = goals.rename(index=str, columns={'Descript6ion' : "SDG"})
goals['total']= goals.stack().groupby(level=0).apply(', '.join)

"""
Include stopwords, tokenize data
"""

def tokenizer(text):
    tokens_ = [word_tokenize(sent) for sent in sent_tokenize(text)]

    tokens = []
    for token_by_sent in tokens_:
        tokens += token_by_sent

    tokens = list(filter(lambda t: t.lower() not in stop, tokens))
    tokens = list(filter(lambda t: t not in punctuation, tokens))
    tokens = list(
        filter(lambda t: t not in [u"'s", u"n't", u"...", u"''", u'``', u'\u2014', u'\u2026', u'\u2013'], tokens))

    filtered_tokens = []
    for token in tokens:
        token = wnl.lemmatize(token)
        if re.search('[a-zA-Z]', token):
            filtered_tokens.append(token)

    filtered_tokens = list(map(lambda token: token.lower(), filtered_tokens))

    return filtered_tokens

class MLStripper(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

# find top keywords (most mentioned)
def get_keywords(tokens, num):
    return Counter(tokens).most_common(num)

def build_article_df(urls):
    articles = []
    for index, row in urls.iterrows():
        try:
            data = row['total'].strip().replace("'", "")
            data = strip_tags(data)
            soup = BeautifulSoup(data, 'html.parser')
            data = soup.get_text()
            data = data.encode('ascii', 'ignore').decode('ascii')
            document = tokenizer(data)
            top_5 = get_keywords(document, 10)

            unzipped = list(zip(*top_5))
            kw = list(unzipped[0])
            kw = ",".join(str(x) for x in kw)
            articles.append((kw, row['total']))
        except Exception as e:
            print(e)
            # print data
            # break
            pass
        # break
    article_df = pd.DataFrame(articles, columns=['keywords', 'total'])
    return article_df

data = goals['total']
data = data.to_frame()
article_df = build_article_df(data)

"""
Analysis - working with own keywords
"""

"""
1. Co-occurence matrix with keywords obtained by highest number of matches (Network map)
"""
keywords_array=[]
for index, row in article_df.iterrows():
    keywords=row['keywords'].split(',')
    for kw in keywords:
        keywords_array.append((kw.strip(' '), row['keywords']))

kw_df = pd.DataFrame(keywords_array).rename(columns={0:'keyword', 1:'keywords'})

document = kw_df.keywords.tolist()
names = kw_df.keyword.tolist()

document_array = []
for item in document:
    items = item.split(',')
    document_array.append((items))

occurrences = OrderedDict((name, OrderedDict((name, 0) for name in names)) for name in names)

for l in document_array:
    for i in range(len(l)):
        for item in l[:i] + l[i + 1:]:
            occurrences[l[i]][item] += 1

co_occur = pd.DataFrame.from_dict(occurrences)

co_occur.to_csv('Co-ocurrence_topKW.csv')

"""
2.. Count Matches
"""
res = []
for a in article_df['total']:
    res.append(strip2.applymap(lambda x: str.count(a, x)))

map = pd.concat(res, axis=1).T
map.index = np.arange(len(map))
map.columns = strip2
cor = map.corr()
# delete columns with zero matches
map = map.loc[:, (map != 0).any(axis=0)]
map.to_csv('correlation_mapping.csv')

"""
3. Co-occurence matrix with own keywords from above
"""
from sklearn.feature_extraction.text import CountVectorizer

samples = article_df['total'].tolist()
keywords = strip2[0].tolist()

bigram_vectorizer = CountVectorizer(ngram_range=(1,2), vocabulary = keywords)
X = bigram_vectorizer.fit_transform(samples)
Xc = (X.T * X)
Xc.setdiag(0)
names = bigram_vectorizer.get_feature_names()
biagram = pd.DataFrame(data = Xc.toarray(), columns = names, index = names)
'''good idea??'''
biagram = biagram.loc[:, (biagram != 0).any(axis=0)]

biagram.to_csv('co_occurenceSDG.csv', sep = ',')
'''Problem: we have way to many meaningless keywords to get a clear picture.'''

"""
4. Apply TextRank to obtain keywords by unsupervised ML (Python 3 needed now)
"""
nlp = spacy.load('en_core_web_sm')

__builtins__.list()
class TextRank4Keyword():
    """Extract keywords from text"""

    def __init__(self):
        self.d = 0.85  # damping coefficient, usually is .85
        self.min_diff = 1e-5  # convergence threshold
        self.steps = 10  # iteration steps
        self.node_weight = None  # save keywords and its weight

    def set_stopwords(self, stopwords):
        """Set stop words"""
        for word in STOP_WORDS.union(set(stopwords)):
            lexeme = nlp.vocab[word]
            lexeme.is_stop = True

    def sentence_segment(self, doc, candidate_pos, lower):
        """Store those words only in cadidate_pos"""
        sentences = []
        for sent in doc.sents:
            selected_words = []
            for token in sent:
                # Store words only with cadidate POS tag
                if token.pos_ in candidate_pos and token.is_stop is False:
                    if lower is True:
                        selected_words.append(token.text.lower())
                    else:
                        selected_words.append(token.text)
            sentences.append(selected_words)
        return sentences

    def get_vocab(self, sentences):
        """Get all tokens"""
        vocab = OrderedDict()
        i = 0
        for sentence in sentences:
            for word in sentence:
                if word not in vocab:
                    vocab[word] = i
                    i += 1
        return vocab

    def get_token_pairs(self, window_size, sentences):
        """Build token_pairs from windows in sentences"""
        token_pairs = list()
        for sentence in sentences:
            for i, word in enumerate(sentence):
                for j in range(i + 1, i + window_size):
                    if j >= len(sentence):
                        break
                    pair = (word, sentence[j])
                    if pair not in token_pairs:
                        token_pairs.append(pair)
        return token_pairs

    def symmetrize(self, a):
        return a + a.T - np.diag(a.diagonal())

    def get_matrix(self, vocab, token_pairs):
        """Get normalized matrix"""
        # Build matrix
        vocab_size = len(vocab)
        g = np.zeros((vocab_size, vocab_size), dtype='float')
        for word1, word2 in token_pairs:
            i, j = vocab[word1], vocab[word2]
            g[i][j] = 1

        # Get Symmeric matrix
        g = self.symmetrize(g)

        # Normalize matrix by column
        norm = np.sum(g, axis=0)
        g_norm = np.divide(g, norm, where=norm != 0)  # this is ignore the 0 element in norm

        return g_norm

    def get_keywords(self, number=30):
        """Print top number keywords"""
        node_weight = OrderedDict(sorted(self.node_weight.items(), key=lambda t: t[1], reverse=True))
        for i, (key, value) in enumerate(node_weight.items()):
            print(key + ' - ' + str(value))
            if i > number:
                break

    def analyze(self, text,
                candidate_pos=['NOUN', 'PROPN'],
                window_size=4, lower=False, stopwords=list()):
        """Main function to analyze text"""

        # Set stop words
        self.set_stopwords(stopwords)

        # Pare text by spaCy
        doc = nlp(text)

        # Filter sentences
        sentences = self.sentence_segment(doc, candidate_pos, lower)  # list of list of words

        # Build vocabulary
        vocab = self.get_vocab(sentences)

        # Get token_pairs from windows
        token_pairs = self.get_token_pairs(window_size, sentences)

        # Get normalized matrix
        g = self.get_matrix(vocab, token_pairs)

        # Initionlization for weight(pagerank value)
        pr = np.array([1] * len(vocab))

        # Iteration
        previous_pr = 0
        for epoch in range(self.steps):
            pr = (1 - self.d) + self.d * np.dot(g, pr)
            if abs(previous_pr - sum(pr)) < self.min_diff:
                break
            else:
                previous_pr = sum(pr)

        # Get weight for each node
        node_weight = dict()
        for word, index in vocab.items():
            node_weight[word] = pr[index]

        self.node_weight = node_weight

text = article_df.total.tolist()
text = ''.join(text)

print('----------------------------------')
print('Ranked Keywords for SDG+Targets+Indicators:')
print('----------------------------------')
tr4w = TextRank4Keyword()
tr4w.analyze(text, candidate_pos = ['NOUN', 'PROPN'], window_size=4, lower=False)
tr4w.get_keywords(30)


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/mukulhase/Dev/github/webwhatsapp-scripts/webwhatsapi/objects/contact.py
# Compiled at: 2018-05-27 06:05:33
# Size of source mod 2**32: 1418 bytes
from six import string_types
from .whatsapp_object import WhatsappObjectWithId, driver_needed
from ..helper import safe_str

class Contact(WhatsappObjectWithId):
    __doc__ = "\n    Class which represents a Contact on user's phone\n    "

    def __init__(self, js_obj, driver=None):
        """

        :param js_obj:
        :param driver:
        :type driver: WhatsAPIDriver
        """
        super(Contact, self).__init__(js_obj, driver)
        if 'shortName' in js_obj:
            self.short_name = js_obj['shortName']
        if 'pushname' in js_obj:
            self.push_name = js_obj['pushname']
        if 'formattedName' in js_obj:
            self.formatted_name = js_obj['formattedName']

    @driver_needed
    def get_common_groups(self):
        return list(self.driver.contact_get_common_groups(self.id))

    @driver_needed
    def get_chat(self):
        return self.driver.get_chat_from_id(self.id)

    def get_safe_name(self):
        """

        :return: String used for representation of the Contact

        :rtype: String

        """
        name = self.name or self.push_name or self.formatted_name
        if isinstance(name, string_types):
            safe_name = safe_str(name)
        else:
            safe_name = 'Unknown'
        return safe_name

    def __repr__(self):
        safe_name = self.get_safe_name()
        return '<Contact {0} ({1})>'.format(safe_name, self.id)


"""
    Station Map
    Keeps reference to all stations
"""

import json
import random
from station import Station

class StationMap(object):

    def get_station(self, number):
        try:
            return self._nodes[number]
        except KeyError:
            new = Station(number)
            self._nodes[number] = new
            return new

    def __init__(self, connections, names):
        self._nodes = {}

        with open(connections, 'r') as stream:
            c = json.load(stream)

        with open(names, 'r') as stream:
            n = json.load(stream)

        for item in c:
            a = self.get_station(int(item[0]))
            b = self.get_station(int(item[1]))
            a.add_connection(b)

        for item in n:
            station = self.get_station(int(item[0]))
            station.name = str(item[1])

    def random(self):
        """ random station from station map """
        while True:
            if not len(self._nodes):
                raise StopIteration

            yield random.choice(list(self._nodes.values()))

    def busiest_station(self):
        values = sorted(
            self._nodes.values(),
            key=lambda x: x.counter,
            reverse=True)

        if len(values):
            return values[0]
        else:
            raise ValueError


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name="home"),
    path('<int:year>/<str:month>', views.home, name="home"),
    path('events', views.all_events, name="event-list"),
    path('add_venue', views.add_venue, name="add-venue"),
    path('list_venues', views.list_venues, name="list-venues"),
    path('show_venue/<int:venue_id>', views.show_venue, name="show-venue"),
    path('contact', views.contact, name="contact"),
]


import os
from datetime import datetime, timedelta

class Config:

    SWAGGER = {
        'title': 'Get Terra Admin',
        'specs_route': '/docs',
        'uiversion': 3,

        'info': {
            'title': 'Get Terra Admin Only API',
            'version': '0.10.0',
            'description': 'Get Terra 서비스의 어드민 전용 API입니다. 현재 개발 및 테스트 중입니다.'
        },
        'host': 'localhost:5000',
        'basePath': '/',
    }

    SWAGGER_TEMPLATE = {
        'schemes': [
            'http'
        ],
        'tags': [
            {
                'name': 'Booth',
                'description': '부스 운영에 관한 API'
            },
            {
                'name': 'Problem',
                'description': '문제에 관한 API'
            },
            {
                'name': 'Icon',
                'description': '각 부스별 고유 QR코드 부여에 관한 API'
            },
            {
                'name': 'Authorization',
                'description': '로그인 및 회원가입에 관한 API'
            },
            {
                'name': 'SerialNumber',
                'description': '플레이어가 게임에 참여할 때 사용되는 인증 코드를 발급하는 API'
            },
            {
                'name': 'Status',
                'description': '부스에 대한 정보를 실시간으로 전달하는 API'
            },
            {
                'name': 'SetUp',
                'description': '팀 설정, 게임 시작/종료 시간을 설정하는 API'
            }
        ]
    }

    JSON_AS_ASCII = False
    JWT_SECRET_KEY = os.getenv('JWT_SECRET_KEY', 'Nerd-Bear')
    SECRET_KEY = os.getenv('SECRET_KEY', 'Nerd-Bear')

    JWT_ACCESS_TOKEN_EXPIRES = timedelta(minutes=30)
    JWT_REFRESH_TOKEN_EXPIRES = timedelta(hours=1)


import collections
from queue import Queue

# Since the graph is undirected, we can start BFS / DFS from any vertex and check if all vertices are reachable or not.
# If all vertices are reachable, then graph is connected, otherwise not.
class Solution:
    def validTree(self, n, edges):
        if len(edges) != n - 1:
            return False

        neighbors = collections.defaultdict(list)
        for u, v in edges:
            neighbors[u].append(v)
            neighbors[v].append(u)

        visited = {}
        queue = Queue()

        queue.put(0)
        visited[0] = True
        while not queue.empty():
            cur = queue.get()
            visited[cur] = True
            for node in neighbors[cur]:
                if node not in visited:
                    visited[node] = True
                    queue.put(node)

        return len(visited) == n

class Solution1:
    def __init__(self):
        self.father = None
        self.size = None

    def vaildTree(self, n, edges):
        if n - 1 != len(edges):
            return False

        self.father = {i: i for i in range(n)}
        self.size = n

        for a, b in edges:
            self.union(a, b)

        return self.size == 1

    def union(self, a, b):
        root_a = self.find(a)
        root_b = self.find(b)
        if root_a != root_b:
            self.size -= 1
            self.father[root_a] = root_b

    def find(self, node):
        path = []
        while node != self.father[node]:
            path.append(node)
            node = self.father[node]

        for n in path:
            self.father[n] = node

        return node


import os
import json
from sqlalchemy import create_engine
import pandas as pd
from sqlalchemy.pool import NullPool

def read_store():
    json_path = os.path.join(os.getcwd(), 'json', 'etl_store.json')
    with open(json_path, 'r') as f:
                store = json.load(f)
    return store

store = read_store()
os.environ["HTTPS_PROXY"] = store["https"]

def check_if_exists(table, schema=''):
    """
    Checks if a table exists in Redshift.
    """
    engine = create_engine(store["redshift"], encoding='utf8', poolclass=NullPool)
    if schema == '':
        table_name = table
        sql_exists = "select * from information_schema.tables where table_name = '{}' ". format(table)
    else:
        table_name = schema + '.' + table
        sql_exists = "select * from information_schema.tables where table_schema = '{}' and table_name = '{}' ". format(schema, table)

    return not pd.read_sql_query(sql = sql_exists, con=engine).empty

def delete_where(table, schema='', *argv):
    """
    Removes records from Redshift table which satisfy *argv.

    Parameters:
    ----------
    table : string
        Name of SQL table.
    schema : string, optional
        Specify the schema.

    Examples:
    --------
        >>> delete_where('test_table', schema='testing', "fiscal_year = '2019'")

        Will generate and execute query:
        "DELETE FROM testing.test WHERE fiscal_year = '2019'"

        >>> delete_where('test_table', schema='testing', "fiscal_year = '2017' OR fiscal_year = '2018'", "customer in ('Enel', 'Agip')")

        Will generate and execute two queries:
        "DELETE FROM testing.test WHERE fiscal_year = '2017' OR fiscal_year = '2018'"
        "DELETE FROM testing.test WHERE customer in ('Enel', 'Agip')"

    """
    table_name = f'{schema}.{table}' if schema else f'{table}'

    if check_if_exists(table, schema):
        engine = create_engine(store["redshift"], encoding='utf8', poolclass=NullPool)

        if argv is not None:
            for arg in argv:
                sql = f"DELETE FROM {table_name} WHERE {arg} "
                engine.execute(sql)
                print(f'Records from table {table_name} where {arg} has been removed successfully.')
    else:
        print(f"Table {table_name} doesn't exist.")


import numpy as np
import cv2
from cv2 import aruco
import glob
import os

def calibrate_charuco():
    aruco_dict = aruco.Dictionary_get(aruco.DICT_6X6_250)
    aruco_params = aruco.DetectorParameters_create()
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)
    board = cv2.aruco.CharucoBoard_create(5, 7, .04, .03, aruco_dict)
    images = glob.glob('../test_footage/calibration_images/*.jpeg')

    all_corners = []
    all_ids = []
    imsize = None

    for fname in images:
        img = cv2.imread(fname)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        corners, ids, rejectedImgPoints = aruco.detectMarkers(gray, aruco_dict, parameters=aruco_params)
        if ids is not None:
            # sub pixel detection
            for i in range(len(corners)):
                corners[i] = cv2.cornerSubPix(gray, corners[i],
                        winSize=(3,3),
                        zeroZone=(-1,-1),
                        criteria=criteria)
            retval, ch_corners, ch_ids = cv2.aruco.interpolateCornersCharuco(
                        corners,
                        ids,
                        gray,
                        board)

            if retval is not None and ch_corners.shape[0] >= 4:
                all_corners.append(ch_corners)
                all_ids.append(ch_ids)
        imsize = gray.shape

    camera_matrix_init = np.array([[ 1000.,    0., imsize[0]/2.],
                              [    0., 1000., imsize[1]/2.],
                              [    0.,    0.,           1.]])

    dist_coeffs_init = np.zeros((5,1))
    flags = (cv2.CALIB_USE_INTRINSIC_GUESS + cv2.CALIB_RATIONAL_MODEL + cv2.CALIB_FIX_ASPECT_RATIO)
    retval, camera_matrix, dist_coeffs, rvecs, tvecs = cv2.aruco.calibrateCameraCharuco(
            charucoCorners=all_corners,
            charucoIds=all_ids,
            board=board,
            imageSize=imsize,
            flags=flags,
            criteria=criteria,
            cameraMatrix=camera_matrix_init,
            distCoeffs=dist_coeffs_init)

    return camera_matrix, dist_coeffs


import torch
import torch.nn as nn
import torch.nn.functional as F

class Event_tagger(nn.Module):
    def __init__(self,
                 word_to_ix,
                 tag_to_ix,
                 ix_to_tag,
                 weights_matrix,
                 batch_size,
                 word_embed_dim=300,
                 bidirectional=True,
                 gru_num_layers=2,
                 dropout=.5):
        '''
            initialize models
            batch_size      - size of batches for traininig
            word_embed_dim  - dimension of word embeddings
            gru_num_layers - number of gru layers
            dropout         - rate for dropout layer
        '''
        super(Event_tagger, self).__init__()

        # dictionaries
        self.word_to_ix = word_to_ix
        self.tag_to_ix = tag_to_ix
        self.ix_to_tag = ix_to_tag

        # parameters
        self.batch_size = batch_size
        self.word_embed_dim = word_embed_dim
        self.gru_hidden_dim = self.word_embed_dim
        self.gru_num_layers = gru_num_layers
        self.dropout = dropout

        self.num_directions = 2 if bidirectional else 1

        self.word_embeds, num_embeds = create_embed_layer(weights_matrix, True)

        self.gru = nn.GRU(self.word_embed_dim,
                          self.gru_hidden_dim,
                          bidirectional=bidirectional,
                          num_layers=self.gru_num_layers,
                          batch_first=True)
        self.dense = nn.Linear(self.gru_hidden_dim * self.num_directions,
                               len(self.tag_to_ix))

    def forward(self, words_batch):
        '''
            words_batch - contains indices of words in current batch with shape
                        (batch_size, num_words_in_sentence)

            creates word level word embeddings from words_batch

            runs Bi-directional gru over input word representation to get
            final word representation which is fed to linear layer and softmax
            activation function to generate probability distribution for event
            tag set
        '''
        # create word-level word embeddings
        word_embed_word_level = words_batch.view(-1)
        word_embed_word_level = self.word_embeds(word_embed_word_level)
        batch_sent_embed = word_embed_word_level.view(
            words_batch.shape[0], -1, word_embed_word_level.shape[-1])

        # create final word representation from gru
        gru_out, _ = self.gru(batch_sent_embed)

        # get probabilities for event tag
        tag_space = self.dense(gru_out)
        tag_scores = F.log_softmax(tag_space, dim=2)
        return tag_scores


import matplotlib.pyplot as plt
import numpy as np
from stompy.spatial import field

import os
opj=os.path.join

from stompy.spatial import interp_coverage
import composite_field_v01

from composite_field_v01 import mbf
dem_dir="tiles_5m_20170501"
os.path.exists(dem_dir) or os.mkdir(dem_dir)

from multiprocessing import Pool

def f(args):
    fn,xxyy,res = args
    if not os.path.exists(fn):
        mbf.to_grid(dx=res,dy=res,bounds=xxyy).write_gdal(fn)

if __name__ == '__main__':
    p = Pool(2)
    res=5.0
    total_bounds=[576000., 596000., 4137000., 4152000.]
    tile_x=tile_y=1000.0

    total_bounds[0] = np.floor(total_bounds[0]/tile_x) * tile_x
    total_bounds[1] = np.ceil(total_bounds[1]/tile_x) * tile_x
    total_bounds[2] = np.floor(total_bounds[2]/tile_y) * tile_y
    total_bounds[3] = np.ceil(total_bounds[3]/tile_y) * tile_y

    calls=[]
    for x0 in np.arange(total_bounds[0],total_bounds[1],tile_x):
        for y0 in np.arange(total_bounds[2],total_bounds[3],tile_y):
            xxyy=(x0,x0+tile_x,y0,y0+tile_y)
            fn=os.path.join(dem_dir,"%.0f_%.0f.tif"%(x0,y0))
            calls.append( [fn,xxyy,res] )

    p.map(f, calls )


import threading
import time
import sys
import random
import pygame

stop_event = threading.Event()
pygame.init()
pygame.mixer.init()
pygame.mixer.music.load("./focusman/1.wav")

def do():
    random.seed()

    start_time = time.time()

    while not stop_event.is_set():
        time.sleep(0.99)
        if random.random() < 0.00055:
            print("A coin!")
            pygame.mixer.music.play(1)

    end_time = time.time()

    print("Time spent: {}".format(round((end_time - start_time) / 60)))

def main():
    calc_thread = threading.Thread(target=do)
    calc_thread.start()

    input("Working...\n")
    stop_event.set()

    calc_thread.join()

if __name__=="__main__":
    main()


import itertools
from scipy.stats import norm
from collections import namedtuple
import math
import numpy as np

from src.chance_constraint.model import CartesianPoint, Vector
from src.chance_constraint.visualization import plot_map

# CartesianPoint = namedtuple('CartesianPoint', 'x y')
# Vector = namedtuple('Vector', 'x y')

def pairwise_circle(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ... (s<last>,s0)"
    a, b = itertools.tee(iterable)
    first_value = next(b, None)
    return itertools.zip_longest(a, b,fillvalue=first_value)

def prob_collision(distance, uncertainty):
    # Survival function (also defined as 1 - cdf, but sf is sometimes more accurate).
    # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.norm.html

    # mi    : média         : location (loc)
    # sigma : desvio padrão : scale

    return norm.sf(distance, loc=0, scale=uncertainty)

def distance_point_line(P, A, B, return_normal=False):
    '''Calculates the distance between the point P and the line that crosses the points A and B'''

    # Director vector of line
    D = Vector((A.x - B.x), (A.y - B.y))

    # Normal vector of line
    #N = Vector(-D.y, D.x)

    # Normalized normal vector of line AB
    aux = (math.sqrt(D.y**2+D.x**2))
    N = Vector( (D.y/aux), ((-D.x)/aux) )

    b = N.x * A.x + N.y * A.y

    distance = P.x * N.x + P.y * N.y - b

    if return_normal:
        return distance, N

    return distance

def chance_constraint(P, obs):
    distances = []

    for A, B in pairwise_circle(obs):
        aux_distance = distance_point_line(P, A, B)
        distances.append(aux_distance)

    distance = max(distances)
    uncertainty = 1 # GPS imprecision

    chance = prob_collision(distance, uncertainty)
    return chance


# Databricks notebook source
# MAGIC %md
# MAGIC ### Fact Inbound Supply Planning (OMNI Planning)
# MAGIC Data going into the Fulfillment Models.
# MAGIC
# MAGIC | Date | Author | Work Item | Notes |
# MAGIC |-----------------------------------|
# MAGIC | 2020-09-01 | Matt Kleiman | OMNI-140382 | Initial notebook based on mapping |

# COMMAND ----------

# DBTITLE 1,Initialization
from pyspark.sql.functions import lit, current_timestamp, expr, col, when, date_format, coalesce
from datetime import datetime, date

# Parameters
dbutils.widgets.text('initFlg', '', '')
dbutils.widgets.text("schmNm", "", "")
dbutils.widgets.text("tblNm", "", "")

# Base variables
schema_name = dbutils.widgets.get("schmNm").lower()
table_name = dbutils.widgets.get("tblNm").lower()
schema_table_name = f'{schema_name}.{table_name}'

table_exists = table_name in sqlContext.tableNames(schema_name)
overwrite = dbutils.widgets.get('initFlg') == 'X'

max_updt_ts = spark.table(schema_table_name).agg({'edw_updt_ts': 'max'}).collect()[0][0] if table_exists and not overwrite else '1900-01-01'

# COMMAND ----------

# DBTITLE 1,Import Shared Functions
# MAGIC %run
# MAGIC /Users/svceimdbrx@columbia.com/edw_admin/shared_functions

# COMMAND ----------

fact_grain = [
  'Sku',
  'MaterialNumber',
  'RegionCode',
  'ReplenishmentFlag',
  'SourcePlanningLocationCode',
  'SourceLocationType',
  'SourceLocationCode',
  'DestinationPlanningLocationCode',
  'DestinationChannel',
  'DestinationSubChannel',
  'TransferType',
  'ProductAvailableFlag',
  'RequestedDeliveryDate',
  'ShipDate',
  'ExpectedDeliveryDate',
  'ScheduledFulfillmentDate',
  'DemandSeasonCode',
  'ExtendedHorizonCode'
]

surrogate_key_list = [
  'DimLocationPlanningKey',
  'DimChannelPlanningKey',
  'DimSubChannelPlanningKey',
  'DimRegionKey',
  'DimProductPlanningKey',
  'DimProductChannelPlanningKey',
  'DimProductSubChannelPlanningKey',
  'DimProductSubChannelSeasonPlanningKey',
  'DimSkuPlanningKey',
  'DimStandardCostKey',
  'DimDemandSeasonKey',
  'DimExtendedHorizonKey',
  'DimFulfillmentSubChannelPlanningKey',
  'DimCurrencySeasonalKey'
]

fact_columns = fact_grain + ['FulfillmentInboundQuantity', 'DimDateKey'] + surrogate_key_list

# COMMAND ----------

# DBTITLE 1,Retrieve Landed Data
# Source tables
base_table = spark.table('entpr_plng_app.planning_inbound_supply_dtc').alias('sup')
planning_location = spark.table('entpr_plng_app.planning_location').alias('reg')
dtc_seas_lkup = spark.table('entpr_foundation_view.omni_dtc_seas_lkup').alias('dseas')
dtc_seas_lkup_extended = dtc_seas_lkup.alias('dext')
product_planning = spark.table('CSC.DimProductPlanning').alias('pp')

# Join conditions
loc_join = col('reg.Lvl_30_Loc_Code') == col('sup.JdaSubRegion')
dtc_seas_lkup_join = expr('sup.JdaFulfillmentSchedDate between dseas.SEAS_STRT_DT and dseas.SEAS_END_DT')
dtc_seas_lkup_join_extended = expr('sup.JdaFulfillmentSchedDate between dext.SEAS_STRT_DT and dext.SEAS_END_DT')
product_join = ((col('pp.EDW_Grain_Type') == lit('MaterialNumber'))
                & (col('pp.ReplenishmentFlag') == lit('N'))
                & (col('pp.EDW_Grain_Key') == col('sup.MaterialNumber'))
                & (col('pp.RegionCode') == col('reg.SubRegion')))

# Combine data
fact_df = (base_table
           .join(
             (planning_location
              .select('SubRegion', 'Lvl_30_Loc_Code')
              .distinct()),
             how='left', on=loc_join
           )
           .join(dtc_seas_lkup, how='left', on=dtc_seas_lkup_join)
           .join(dtc_seas_lkup_extended, how='left', on=dtc_seas_lkup_join_extended)
           .join(product_planning, how='left', on=product_join)
           .select('sup.*', 'reg.*', 'pp.MaterialNumberSeasonHistory', 'dseas.SEAS_CD', 'dext.SEAS_CD')
           .withColumn('Sku', col('SkuKey'))
           .withColumn('MaterialNumber', col('MaterialNumber'))
           .withColumn('RegionCode', col('SubRegion'))
           .withColumn('ReplenishmentFlag', lit('N'))
           .withColumn('SourcePlanningLocationCode', col('JdaSourceLoc'))
           .withColumn('SourceLocationType', expr("case when substring(TransferType, 1, 5)='STORE' then 'Store' else 'Plant' end"))
           .withColumn('SourceLocationCode', expr("case when substring(TransferType, 1, 5)='STORE' then ShippingStoreNumber else 'Plant' end"))
           .withColumn('DestinationPlanningLocationCode', col('JdaFulfillmentOmniLoc'))
           .withColumn('DestinationChannel', lit('DTC'))
           .withColumn('DestinationSubChannel', col('StoreType'))
           .withColumn('TransferType', col('TransferType'))
           .withColumn('ProductAvailableFlag', col('ProductAvailableFlag'))
           .withColumn('RequestedDeliveryDate', col('RequestedDeliveryDate'))
           .withColumn('ShipDate', col('ShipDate'))
           .withColumn('ExpectedDeliveryDate', col('ExpectedDeliveryDate'))
           .withColumn('ScheduledFulfillmentDate', col('JdaFulfillmentSchedDate'))
           .withColumn('DimDateKey', date_format(col('ScheduledFulfillmentDate'), 'yyyyMMdd').cast('int'))
           .withColumn('DemandSeasonCode', coalesce(expr('regexp_extract(pp.MaterialNumberSeasonHistory, dseas.SEAS_CD, 0)'), lit('')))
           .withColumn('ExtendedHorizonCode',
                       when(
                         ((col('dext.SEAS_CD').substr(1, 1) == lit('S')) & ~(col('pp.MaterialNumberSeasonHistory').like('%F%'))),
                         coalesce(expr('regexp_extract(pp.MaterialNumberSeasonHistory, dext.SEAS_CD, 0)'), lit('')))
                       .when(
                         ((col('dext.SEAS_CD').substr(1, 1) == lit('F')) & ~(col('pp.MaterialNumberSeasonHistory').like('%S%'))),
                         coalesce(expr('regexp_extract(pp.MaterialNumberSeasonHistory, dext.SEAS_CD, 0)'), lit('')))
                       .otherwise(lit('')))
           .groupBy(*fact_grain, 'DimDateKey')
           .agg({'JdaFulfillmentInboundQuantity': 'sum'})
           .withColumn('FulfillmentInboundQuantity', col('sum(JdaFulfillmentInboundQuantity)').cast('decimal(19,8)'))
           .add_surrogate_keys(surrogate_key_list)
           .select(*fact_columns)
           .add_edw_fields()
          )

# COMMAND ----------

# DBTITLE 1,Update Target Table
arg_dict = {'name': schema_table_name, 'format': 'delta', 'mode': 'overwrite'}

if overwrite or not table_exists:
  arg_dict['mergeSchema'] = 'true'
  spark.sql(f'drop table if exists {schema_table_name}')
  dbutils.fs.rm(f'/mnt/entadls/published/eim/managed/{schema_name}/{table_name}', True)
  print('Overwriting...')

fact_df.write.saveAsTable(**arg_dict)

# COMMAND ----------

# DBTITLE 1,Write to ADW
spark.table(schema_table_name).filter(f"EDW_UPDT_TS > '{max_updt_ts}'").createOrReplaceGlobalTempView(table_name)

dbutils.notebook.run('/Users/svceimdbrx@columbia.com/edw_admin/adw_integration_write', 120000, {'schemaNm': f'{schema_name.upper()}_LND', 'tableNm': table_name, 'dbrxTable': table_name, 'writeMode': 'overwrite'})

spark.sql(f'drop view global_temp.{table_name}')


# coding=utf-8
# ------------------------------------
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# ------------------------------------

import functools
from asynctestcase import AsyncDocumentTranslationTest
from preparer import DocumentTranslationPreparer, DocumentTranslationClientPreparer as _DocumentTranslationClientPreparer
from azure.ai.translation.document.aio import DocumentTranslationClient
import pytest
DocumentTranslationClientPreparer = functools.partial(_DocumentTranslationClientPreparer, DocumentTranslationClient)

TOTAL_DOC_COUNT_IN_JOB = 1

class TestSubmittedJobs(AsyncDocumentTranslationTest):

    @DocumentTranslationPreparer()
    @DocumentTranslationClientPreparer()
    async def test_list_submitted_jobs(self, client):
        # prepare data
        jobs_count = 3

        # create some jobs
        job_ids = await self._create_and_submit_sample_translation_jobs_async(client, jobs_count)

        # list jobs
        submitted_jobs = client.list_submitted_jobs()
        self.assertIsNotNone(submitted_jobs)

        # check statuses
        jobs_list = []
        async for job in submitted_jobs:
            jobs_list.append(job)
            if job.id in job_ids:
                self._validate_translation_job(job, status="Succeeded", total=TOTAL_DOC_COUNT_IN_JOB, succeeded=TOTAL_DOC_COUNT_IN_JOB)
            else:
                self._validate_translation_job(job)

    @pytest.mark.skip("top not exposed yet")
    @DocumentTranslationPreparer()
    @DocumentTranslationClientPreparer()
    async def test_list_submitted_jobs_with_pagination(self, client):
        # prepare data
        jobs_count = 6
        result_per_page = 2
        no_of_pages = jobs_count // result_per_page

        # create some jobs
        job_ids = await self._create_and_submit_sample_translation_jobs_async(client, jobs_count)

        # list jobs
        submitted_jobs_pages = client.list_submitted_jobs(results_per_page=result_per_page)
        pages_list = []

        # iterate by page
        async for page in submitted_jobs_pages:
            pages_list.append()
            page_jobs = []

            async for job in page:
                page_jobs.append(job)
                if job.id in job_ids:
                    self._validate_translation_job(job, status="Succeeded", total=TOTAL_DOC_COUNT_IN_JOB, succeeded=TOTAL_DOC_COUNT_IN_JOB)
                else:
                    self._validate_translation_job(job)

            self.assertEqual(len(page_jobs), result_per_page)

    @pytest.mark.skip("skip not exposed yet")
    @DocumentTranslationPreparer()
    @DocumentTranslationClientPreparer()
    async def test_list_submitted_jobs_with_skip(self, client):
        # prepare data
        jobs_count = 6
        skip = 2

        # create some jobs
        job_ids = await self._create_and_submit_sample_translation_jobs_async(client, jobs_count)

        # list jobs
        submitted_jobs = client.list_submitted_jobs(skip=skip)
        jobs_list = []

        async for job in submitted_jobs:
            jobs_list.append(job)
            if job.id in job_ids:
                self._validate_translation_job(job, status="Succeeded", total=TOTAL_DOC_COUNT_IN_JOB, succeeded=TOTAL_DOC_COUNT_IN_JOB)
            else:
                self._validate_translation_job(job)


import sys
import mechanize
import util
from BeautifulSoup import BeautifulSoup

browser = None
uni_url = None
user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0'
planet_name = None

def login(config):
    global browser
    global uni_url
    global planet_name
    browser = mechanize.Browser()
    browser.set_handle_robots(False)
    browser.set_handle_refresh(False)
    browser.addheaders = [('User-agent', user_agent)]
    lang = config.get('credentials', 'universe_language')
    uni_id = config.get('credentials', 'universe_id')
    uni_url = 's%s-%s.ogame.gameforge.com' % (uni_id, lang)
    base_url = 'http://%s.ogame.gameforge.com/' % lang

    request = mechanize.Request(base_url)
    browser.open(request)
    browser.select_form('loginForm')
    control = browser.form.find_control('uni')
    for item in control.items:
        if item.name == uni_url:
            item.selected = True
            break
    user = config.get('credentials', 'username')
    browser.form['login'] = user
    browser.form['pass'] = config.get('credentials', 'password')
    browser.submit()
    uni_name = config.get('credentials', 'universe_name')
    if uni_name in browser.title():
        util.log('Logged as %s ' % user + 'in %s' % uni_name)
    else:
        util.log('Login error: %s' % browser.geturl())
        sys.exit(1)

    soup = BeautifulSoup(browser.response())
    planets = soup.find(id='planetList').findAll('div')

    selected_planet = planets[int(config.get('credentials', 'planet'))]
    new_planet = selected_planet.find('a')
    planet_name = new_planet.find('span').getText()
    browser.open(new_planet['href'])


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name='luciferase',
    version='2.4.3',
    author='Anthony Aylward, Joshua Chiou, Mei-Lin Okino',
    author_email='aaylward@eng.ucsd.edu',
    description='Helper functions for luciferase data',
    long_description=long_description,
    long_description_content_type="text/markdown",
    url='https://github.com/anthony-aylward/luciferase.git',
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent"
    ],
    install_requires=[
        'estimateratio', 'matplotlib', 'pandas', 'scipy', 'seaborn', 'xlrd'
    ],
    entry_points={
        'console_scripts': [
            'luciferase-barplot=luciferase.luciferase:main',
            'luciferase-ratioplot=luciferase.ratioplot:main',
            'luciferase-swarmplot=luciferase.swarmplot:main',
            'luciferase-ratiotest=luciferase.ratiotest:main',
            'luciferase-write-table=luciferase.write_table:main'
        ]
    }
)


from spaceone.inventory.libs.manager import GoogleCloudManager
from spaceone.inventory.libs.schema.base import ReferenceModel
from spaceone.inventory.model.disk.data import *
from spaceone.inventory.model.disk.cloud_service import *
from spaceone.inventory.connector.disk import DiskConnector
from spaceone.inventory.model.disk.cloud_service_type import CLOUD_SERVICE_TYPES
from datetime import datetime
import time

class DiskManager(GoogleCloudManager):
    connector_name = 'DiskConnector'
    cloud_service_types = CLOUD_SERVICE_TYPES

    def collect_cloud_service(self, params):
        print("** Disk START **")
        start_time = time.time()
        """
        Args:
            params:
                - options
                - schema
                - secret_data
                - filter
                - zones
        Response:
            CloudServiceResponse
        """
        secret_data = params['secret_data']
        disk_conn: DiskConnector = self.locator.get_connector(self.connector_name, **params)
        disks = disk_conn.list_disks()
        resource_policies = disk_conn.list_resource_policies()

        collected_cloud_services = []

        for disk in disks:
            disk_type = self._get_last_target(disk.get('type'))
            disk_size = float(disk.get('sizeGb'))
            zone = self._get_last_target(disk.get('zone'))
            region = zone[:-2]

            labels = self.convert_labels_format(disk.get('labels', {}))
            disk.update({
                'project': secret_data['project_id'],
                'zone': zone,
                'region': region,
                'in_used_by': self._get_in_used_by(disk.get('users', [])),
                'source_image_display': self._get_source_image_display(disk),
                'disk_type': disk_type,
                'snapshot_schedule': self.get_matched_snapshot(region, disk, resource_policies),
                'snapshot_schedule_display': self._get_snapshot_schedule(disk),
                'encryption': self._get_encryption(disk),
                'size': float(self._get_bytes(int(disk.get('sizeGb')))),
                'read_iops': self.get_iops_rate(disk_type, disk_size, 'read'),
                'write_iops': self.get_iops_rate(disk_type, disk_size, 'write'),
                'read_throughput': self.get_throughput_rate(disk_type, disk_size),
                'write_throughput': self.get_throughput_rate(disk_type, disk_size),
                'labels': labels
            })

            disk_data = Disk(disk, strict=False)
            disk_resource = DiskResource({
                'tags': labels,
                'data': disk_data,
                'region_code': disk['region'],
                'reference': ReferenceModel(disk_data.reference())
            })

            self.set_region_code(disk['region'])
            collected_cloud_services.append(DiskResponse({'resource': disk_resource}))

        print(f'** Disk Finished {time.time() - start_time} Seconds **')
        return collected_cloud_services

    def get_iops_rate(self, disk_type, disk_size, flag):
        const = self._get_iops_constant(disk_type, flag)
        return disk_size * const

    def get_throughput_rate(self, disk_type, disk_size):
        const = self._get_throughput_constant(disk_type)
        return disk_size * const

    def get_matched_snapshot(self, region, disk, resource_policies):
        matched_policies = []
        policy_self_links = disk.get('resourcePolicies', [])
        policies = resource_policies.get(region)

        for self_link in policy_self_links:
            for policy in policies:
                if policy.get('selfLink') == self_link:
                    snapshot_schedule_policy = policy.get('snapshotSchedulePolicy', {})
                    snapshot_prop = snapshot_schedule_policy.get('snapshotProperties', {})
                    retention = snapshot_schedule_policy.get('retentionPolicy', {})
                    retention.update({'max_retention_days_display': str(retention.get('maxRetentionDays')) + ' days'})
                    policy_schedule = snapshot_schedule_policy.get('schedule', {})

                    policy.update({
                        'snapshot_schedule_policy': {
                            'schedule_display': self._get_schedule_display(policy_schedule),
                            'schedule': policy_schedule,
                            'retention_policy': retention
                        },
                        'region': self._get_last_target(policy.get('region')),
                        'labels': self.convert_labels_format(snapshot_prop.get('labels', {})),
                        'tags': self.convert_labels_format(snapshot_prop.get('labels', {})),
                        'storage_locations': snapshot_prop.get('storageLocations', [])
                    })
                    matched_policies.append(policy)

        return matched_policies

    def _get_schedule_display(self, schedule):
        schedule_display = []
        if 'weeklySchedule' in schedule:
            week_schedule = schedule.get('weeklySchedule', {})
            weeks = week_schedule.get('dayOfWeeks', [])
            for week in weeks:
                schedule_display.append(week.get('day').title() + self._get_readable_time(week))

        elif 'dailySchedule' in schedule:
            daily = schedule.get('dailySchedule')
            schedule_display.append(f'Every day{self._get_readable_time(daily)}')

        elif 'hourlySchedule' in schedule:
            hourly = schedule.get('hourlySchedule')
            cycle = str(hourly.get('hoursInCycle'))
            hourly_schedule = f'Every {cycle} hours'
            schedule_display.append(hourly_schedule)

        return schedule_display

    @staticmethod
    def _get_readable_time(day_of_weeks):
        start_time = day_of_weeks.get('startTime')
        time_frame = start_time.split(':')
        first = int(time_frame[0]) + 1
        second = int(time_frame[1])

        d = datetime.strptime(start_time, "%H:%M")
        start = d.strftime("%I:%M %p")
        e = datetime.strptime(f'{first}:{second}', "%H:%M")
        end = e.strftime("%I:%M %p")

        return f' between {start} and {end}'

    @staticmethod
    def _get_iops_constant(disk_type, flag):
        constant = 0
        if flag == 'read':
            if disk_type == 'pd-standard':
                constant = 0.75
            elif disk_type == 'pd-balanced':
                constant = 6.0
            elif disk_type == 'pd-ssd':
                constant = 30.0
        else:
            if disk_type == 'pd-standard':
                constant = 1.5
            elif disk_type == 'pd-balanced':
                constant = 6.0
            elif disk_type == 'pd-ssd':
                constant = 30.0
        return constant

    @staticmethod
    def _get_throughput_constant(disk_type):
        constant = 0
        if disk_type == 'pd-standard':
            constant = 0.12
        elif disk_type == 'pd-balanced':
            constant = 0.28
        elif disk_type == 'pd-ssd':
            constant = 0.48

        return constant

    @staticmethod
    def _get_source_image_display(disk):
        source_image_display = ''
        source_image = disk.get('sourceImage')
        if source_image:
            source_image_display = source_image[source_image.rfind('/') + 1:]
        return source_image_display

    @staticmethod
    def _get_snapshot_schedule(disk):
        snapshot_schedule = []
        policies = disk.get('resourcePolicies', [])
        for policy in policies:
            snapshot_schedule.append(policy[policy.rfind('/') + 1:])
        return snapshot_schedule

    @staticmethod
    def _get_in_used_by(users):
        in_used_by = []
        for user in users:
            used_single = user[user.rfind('/') + 1:]
            in_used_by.append(used_single)
        return in_used_by

    @staticmethod
    def _get_bytes(number):
        return 1024 * 1024 * 1024 * number

    @staticmethod
    def _get_encryption(disk):
        encryption = 'Google managed'
        disk_encryption = disk.get('diskEncryptionKey')
        if disk_encryption:
            if 'kmsKeyName' in disk_encryption or 'kmsKeyServiceAccount' in disk_encryption:
                encryption = 'Customer managed'
            else:
                encryption = 'Customer supplied'
        return encryption

    @staticmethod
    def _get_last_target(target):
        return target[target.rfind('/') + 1:]


#!/usr/bin/python

import json
import sys

sample_size = 1000
sample_count = 0

threshold = 100

inputfile = open('./statistics/agents.json', 'r')

data = inputfile.read()

obj = json.loads(data)

agents = []
os_lookup = {}
os_lookup_id = 0
os_name_lookup = {}
os_name_lookup_id = 0
name_lookup = {}
name_lookup_id = 0

unique_agents_count = 0
agents_count = 0
filtered_count = 0

for value in obj:
    count = value['count']
    value.pop('count', None)

    if 'major' in value['useragent']:
        del value['useragent']['major']

    if 'minor' in value['useragent']:
        del value['useragent']['minor']

    if 'patch' in value['useragent']:
        del value['useragent']['patch']

    if 'device' in value['useragent']:
        del value['useragent']['device']

    if 'os_major' in value['useragent']:
        del value['useragent']['os_major']

    if 'os_minor' in value['useragent']:
        del value['useragent']['os_minor']

    if 'build' in value['useragent']:
        del value['useragent']['build']

    if count > threshold and sample_count < sample_size:
        sample_count +=1
        agents_count += count
        unique_agents_count += 1

        item = [value['agent']]

        if 'os' in value['useragent']:
            if value['useragent']['os'] in os_lookup:
                item.append(os_lookup[value['useragent']['os']])
            else:
                os_lookup[value['useragent']['os']] = str(os_lookup_id)
                item.append(str(os_lookup_id))
                os_lookup_id += 1
        else:
            item.append('')

        if 'os_name' in value['useragent']:
            if value['useragent']['os_name'] in os_name_lookup:
                item.append(os_name_lookup[value['useragent']['os_name']])
            else:
                os_name_lookup[value['useragent']['os_name']] = str(os_name_lookup_id)
                item.append(str(os_name_lookup_id))
                os_name_lookup_id += 1
        else:
            item.append('')

        if 'name' in value['useragent']:
            if value['useragent']['name'] in name_lookup:
                item.append(name_lookup[value['useragent']['name']])
            else:
                name_lookup[value['useragent']['name']] = str(name_lookup_id)
                item.append(str(name_lookup_id))
                name_lookup_id += 1
        else:
            item.append('')

        agents.append([count, item])

    else:
        filtered_count += count

# Reverse lookups
os_lookup = {v: k for k, v in os_lookup.items()}
os_name_lookup = {v: k for k, v in os_name_lookup.items()}
name_lookup = {v: k for k, v in name_lookup.items()}

outputfile = open('./config/agents_data_sample.js', 'w')

outputfile.write("/* Items in the agents list has the following structure: [<count>, [<agent>, <useragent.os lookup id>, <useragent.os_name lookup id>, <useragent.name lookup id>]] */\n\n")
outputfile.write("module.exports.agents = ")
outputfile.write(json.dumps(agents, separators=(',', ':')))
outputfile.write(";\n\nmodule.exports.os_lookup = ")
outputfile.write(json.dumps(os_lookup, separators=(',', ':')))
outputfile.write(";\n\nmodule.exports.os_name_lookup = ")
outputfile.write(json.dumps(os_name_lookup, separators=(',', ':')))
outputfile.write(";\n\nmodule.exports.name_lookup = ")
outputfile.write(json.dumps(name_lookup, separators=(',', ':')))
outputfile.write(";")

outputfile.close()

total = agents_count + filtered_count

sys.stdout.write('Unique agent count: {}\n'.format(unique_agents_count))
sys.stdout.write('Agent count: {} ({})\n'.format(agents_count, agents_count * 100.0 / total))
sys.stdout.write('Filtered count: {} ({})\n'.format(filtered_count, filtered_count * 100.0 / total))


import logging

from API.api.base_api import BaseApi

class User(BaseApi):
    '''
    成员管理
    '''

    def create_user(self,data):
        '''
        创建的用户
        :param data:
        :return:
        '''
        userdata=data
        r=self.request('post',f'user/create?access_token={self.token}',json=userdata)
        logging.info(f'{r.json()}')
        return r

    def read_user(self,userid):
        '''
        获取用户
        :return:
        '''
        r=self.request("get",f'user/get?access_token={self.token}&userid={userid}')
        logging.info(f'{r.json()}')
        return r
    def update_user(self,data):
        '''
        更新用户
        :param data:
        :return:
        '''
        updata=data
        r=self.request('post',f'user/update?access_token={self.token}',json=updata)
        logging.info(f'{r.json()}')
        return r
    def delete_user(self,userid):
        '''
        删除用户
        :return:
        '''
        r=self.request('get',f'user/delete?access_token={self.token}&userid={userid}')
        logging.info(f'{r.json()}')
        return r


# -*- coding: utf-8 -*-
import argparse
import os
import shutil
import time

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import math
import models
import utils
import copy

model_names = sorted(name for name in models.__dict__
                     if name.islower() and not name.startswith("__")
                     and callable(models.__dict__[name]))
parser = argparse.ArgumentParser(description='Implement DPSS by PyTorch')

parser.add_argument('--data-dir', type=str, default='./data', help='path to dataset')
parser.add_argument('--dataset', type=str, default='cifar10', choices=['cifar10', 'cifar100', 'ILSVRC2012'],
                    help='training dataset (default: cifar10)')
parser.add_argument('--save-dir', type=str, default='./snapshot', help='Folder to save checkpoints.')
parser.add_argument('--model', type=str, default='', metavar='PATH', help='import model(default: none)')
parser.add_argument('--arch', type=str, metavar='ARCH', default='vggsmall',
                    choices=model_names,
                    help='model architecture: ' +
                         ' | '.join(model_names) +
                         ' (default: resnet18)')
parser.add_argument('--lambda21', type=float, default=0.01, help='L21 regularization')
parser.add_argument('--flops', action='store_true', help='use pruned flops')
parser.add_argument('--pr', type=float, default=0.94, help='pruned ratio')
parser.add_argument('-stsr', action='store_true', help='stop progressive regulation')

parser.add_argument('-j', '--workers', default=1, type=int, metavar='N',
                    help='number of data loading workers (default: 1)')
parser.add_argument('--epochs', default=200, type=int, metavar='N', help='number of total epochs to run')
parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                    help='manual epoch number (useful on restarts)')
parser.add_argument('--batch-size', default=64, type=int,
                    metavar='N', help='mini-batch size (default: 128)')
parser.add_argument('--test-batch-size', type=int, default=100, metavar='N',
                    help='input batch size for testing (default: 100)')
parser.add_argument('--lr', default=0.1, type=float,
                    metavar='LR', help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum')
parser.add_argument('--weight-decay', default=1e-4, type=float,
                    metavar='W', help='weight decay (default: 1e-4)')
parser.add_argument('--print-freq', default=100, type=int,
                    metavar='N', help='print frequency (default: 100)')
parser.add_argument('--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
                    help='evaluate model on validation set')
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',
                    help='random seed (default: 1)')

best_prec1 = 0

def main():
    global args, best_prec1, log, log1, log2, log3
    args = parser.parse_args()
    args.save_time = args.arch + '_experiment_' + args.dataset + '_' + '%.4f' % (args.lambda21) + '_' + str(args.flops) + '_flops_' + '%.3f' % (
    args.pr) + '_' + '%d' % (args.epochs)
    args.save_dir = os.path.join('./snapshot/', args.dataset + '_experiment_' + args.arch + '_' + 'lambda21' + '_' + '%d' % (args.epochs))
    args.save_time = args.save_time + '_dpss_sigmoid_filter_sum_context_add_scratch_' + time.strftime("%Y%m%d%H%M", time.localtime())
    if not os.path.isdir(args.save_dir):
        os.makedirs(args.save_dir)
    log = open(os.path.join(args.save_dir, '{}.log'.format(args.save_time)), 'w')
    log1 = open(os.path.join(args.save_dir, '{}_rate_epoch.txt'.format(args.save_time)), 'w')
    log2 = open(os.path.join(args.save_dir, '{}_sparsity_ratio_epoch.txt'.format(args.save_time)), 'w')
    log3 = open(os.path.join(args.save_dir, '{}_sparsity_allocation_ratio_epoch.txt'.format(args.save_time)), 'w')
    model_save_name = os.path.join(args.save_dir, args.save_time)
    args.cuda = not args.no_cuda and torch.cuda.is_available()
    print_log('Parameters setting: \n''epochs: {}\t''lr: {}\n'.format(args.epochs, args.lr), log)

    # initial
    torch.manual_seed(args.seed)
    if args.cuda:
        torch.cuda.manual_seed(args.seed)

    # Data loading code
    kwargs = {'num_workers': args.workers, 'pin_memory': True} if args.cuda else {}
    if args.dataset == 'cifar10':
        train_loader = torch.utils.data.DataLoader(
            datasets.CIFAR10(args.data_dir, train=True, download=True,
                             transform=transforms.Compose([
                                 transforms.Pad(4),
                                 transforms.RandomCrop(32),
                                 transforms.RandomHorizontalFlip(),
                                 transforms.ToTensor(),
                                 transforms.Normalize([0.4914, 0.4824, 0.4467], [0.2471, 0.2435, 0.2616])
                             ])),
            batch_size=args.batch_size, shuffle=True, **kwargs)
        val_loader = torch.utils.data.DataLoader(
            datasets.CIFAR10(args.data_dir, train=False, transform=transforms.Compose([
                transforms.ToTensor(),
                transforms.Normalize([0.4914, 0.4824, 0.4467], [0.2471, 0.2435, 0.2616])
            ])),
            batch_size=args.test_batch_size, shuffle=True, **kwargs)
        num_classes = 10
        input_pix = 32

    elif args.dataset == 'cifar100':
        train_loader = torch.utils.data.DataLoader(
            datasets.CIFAR100(args.data_dir, train=True, download=True,
                              transform=transforms.Compose([
                                  transforms.Pad(4),
                                  transforms.RandomCrop(32),
                                  transforms.RandomHorizontalFlip(),
                                  transforms.ToTensor(),
                                  transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761])
                              ])),
            batch_size=args.batch_size, shuffle=True, **kwargs)
        val_loader = torch.utils.data.DataLoader(
            datasets.CIFAR100(args.data_dir, train=False, transform=transforms.Compose([
                transforms.ToTensor(),
                transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761])
            ])),
            batch_size=args.test_batch_size, shuffle=True, **kwargs)
        num_classes = 100
        input_pix = 32
    else:
        train_loader = torch.utils.data.DataLoader(
            datasets.ImageFolder(os.path.join(args.data_dir, 'train'),
                                 transforms.Compose([
                                     # transforms.Scale(256),
                                     transforms.RandomResizedCrop(224),
                                     transforms.RandomHorizontalFlip(),
                                     transforms.ToTensor(),
                                     transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
                                 ])),
            batch_size=args.batch_size, shuffle=True, **kwargs)
        val_loader = torch.utils.data.DataLoader(
            datasets.ImageFolder(os.path.join(args.data_dir, 'val'),
                                 transforms.Compose([
                                     transforms.Resize(256),
                                     transforms.CenterCrop(224),
                                     transforms.ToTensor(),
                                     transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
                                 ])),
            batch_size=args.test_batch_size, shuffle=True, **kwargs)
        num_classes = 1000
        input_pix = 224

    # create model
    model = models.__dict__[args.arch]()
    if args.cuda:
        model.cuda()

    # multi-gpu
    gpu_num = torch.cuda.device_count()
    print_log('GPU NUM: {:2d}'.format(gpu_num), log)
    model = torch.nn.DataParallel(model, list(range(gpu_num))).cuda()
    # if gpu_num > 1:
    #     model = torch.nn.DataParallel(model, list(range(gpu_num))).cuda()

    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda()

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '{}'".format(args.resume))
            model.load_state_dict(torch.load(args.resume))
        else:
            print("=> no checkpoint found at '{}'".format(args.resume))

    cudnn.benchmark = True

    if args.evaluate:
        validate(val_loader, model, criterion)
        return

    method = DPSS(model, args.lambda21, args.pr)
    model_pruning = copy.deepcopy(model)
    params_before = utils.print_model_param_nums(model_pruning)
    flops_before = utils.count_model_param_flops(model_pruning, input_pix)
    validate(val_loader, model, criterion)
    optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
    next_ratio = 1.
    for epoch in range(args.start_epoch, args.epochs):
        adjust_learning_rate(optimizer, epoch)
        model_pruning = copy.deepcopy(model)
        method1 = DPSS(model_pruning, args.lambda21, args.pr)
        # method1.adjust_scale_coe(next_ratio)
        method1.channel_prune()
        print_log(str(method1.layer_sparsity_ratio), log2)
        params_pruning = utils.print_model_param_nums(method1.model.module)
        flops_pruning = utils.count_model_param_flops(method1.model.module, input_pix)
        pruned_params = 1 - params_pruning / params_before
        pruned_flops = 1 - flops_pruning / flops_before
        if not args.flops:
            next_ratio = get_new_scale(args.pr, pruned_params, copy.deepcopy(model), input_pix, params_before, next_ratio)
        else:
            next_ratio = get_new_scale(args.pr, pruned_flops, copy.deepcopy(model), input_pix, flops_before, next_ratio, args.flops)
        method.adjust_scale_coe(next_ratio)
        model_pruning1 = copy.deepcopy(model)
        method2 = DPSS(model_pruning1, args.lambda21, args.pr)
        method2.adjust_scale_coe(next_ratio)
        method2.channel_prune()
        print_log(str(method2.layer_sparsity_allocation_ratio), log3)
        print(next_ratio)
        train_loss = train(train_loader, model, criterion, optimizer, epoch, method, True)
        # evaluate on validation set
        prec1 = validate(val_loader, model, criterion)
        # remember best prec@1 and save checkpoint
        is_best = prec1 > best_prec1
        best_prec1 = max(prec1, best_prec1)
        save_checkpoint(model, model_save_name, is_best)

        print_log('{}\t''{trn_loss:.4f}\t''{tst_acc:.4f}\t''{pratio:.4f}\t''{fratio:.4f}\t'.format(epoch, trn_loss=train_loss, tst_acc=prec1, pratio=(params_before - params_pruning) / params_before, fratio=(flops_before - flops_pruning) / flops_before), log1)
        print_log('{}\t''{trn_loss:.4f}\t''{tst_acc:.4f}\t''{pratio:.4f}\t''{fratio:.4f}\t'.format(epoch, trn_loss=train_loss, tst_acc=prec1, pratio=(params_before - params_pruning) / params_before, fratio=(flops_before - flops_pruning) / flops_before), log)

    print_log('Get new model\n', log)
    model = torch.load(model_save_name + '_best.pth.tar')
    if args.cuda:
        model.cuda()
    method2 = DPSS(model, args.lambda21, args.pr)
    method2.channel_prune()
    model = method2.model
    if args.cuda:
        model.cuda()
    validate(val_loader, model, criterion)
    save_checkpoint(model, model_save_name + '_pruned', False)
    params_after = utils.print_model_param_nums(model.module)
    flops_after = utils.count_model_param_flops(model.module, input_pix)
    print_log('Before pruning: \n'
          'params: {}\t''flops: {}\n'
          'After pruning: \n'
          'params: {}\t''flops: {}\n'
          'params_ratio: {pratio:.2f}%\t''flops_ratio: {fratio:.2f}%\n'
          'params_rate: {prate:.2f}\t''flops_rate: {frate:.2f}\n'
          'Prec@1: {top1:.4f}'.format(
        params_before, flops_before, params_after, flops_after,
        pratio=(params_before - params_after) * 100. / params_before,
        fratio=(flops_before - flops_after) * 100. / flops_before,
        prate=params_before / params_after,
        frate=flops_before / flops_after,
        top1=best_prec1), log)
    log.close()
    log1.close()
    log2.close()
    log3.close()

def get_new_scale(dest_pr, input_pr, input_model, input_pix, input_before, input_ratio=1., flops=False):
    e_pr = dest_pr
    v_pr = input_pr
    get_ratio = input_ratio

    if abs(v_pr - e_pr) < 0.002:
        get_ratio = 1.0
    else:
        model_pruning = copy.deepcopy(input_model)
        method1 = DPSS(model_pruning, args.lambda21, args.pr)
        method1.adjust_scale_coe(get_ratio)
        method1.channel_prune()
        model_pruning = method1.model
        params_pruning = utils.print_model_param_nums(model_pruning.module)
        flops_pruning = utils.count_model_param_flops(model_pruning.module, input_pix)
        if not flops:
            v_pr = 1 - params_pruning / input_before
        else:
            v_pr = 1 - flops_pruning / input_before
        while v_pr > e_pr:
            get_ratio += 0.005
            model_pruning = copy.deepcopy(input_model)
            method1 = DPSS(model_pruning, args.lambda21, args.pr)
            method1.adjust_scale_coe(get_ratio)
            method1.channel_prune()
            model_pruning = method1.model
            params_pruning = utils.print_model_param_nums(model_pruning.module)
            flops_pruning = utils.count_model_param_flops(model_pruning.module, input_pix)
            if not flops:
                v_pr = 1 - params_pruning / input_before
            else:
                v_pr = 1 - flops_pruning / input_before
        else:
            while v_pr < e_pr:
                get_ratio -= 0.005
                model_pruning = copy.deepcopy(input_model)
                method1 = DPSS(model_pruning, args.lambda21, args.pr)
                method1.adjust_scale_coe(get_ratio)
                method1.channel_prune()
                model_pruning = method1.model
                params_pruning = utils.print_model_param_nums(model_pruning.module)
                flops_pruning = utils.count_model_param_flops(model_pruning.module, input_pix)
                if not flops:
                    v_pr = 1 - params_pruning / input_before
                else:
                    v_pr = 1 - flops_pruning / input_before

    return get_ratio

def train(train_loader, model, criterion, optimizer, epoch, method, method_flag=False):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to train mode
    model.train()

    end = time.time()
    s_e = sparse_coefficent(epoch, args)
    method.sparse_coefficent_value(s_e, args.stsr)
    for i, (data, target) in enumerate(train_loader):
        # measure data loading time
        data_time.update(time.time() - end)

        if args.cuda:
            data, target = data.cuda(), target.cuda()

        # compute output
        output = model(data)
        loss = criterion(output, target)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.item(), data.size(0))
        top1.update(prec1[0], data.size(0))
        top5.update(prec5[0], data.size(0))

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        # if i % 5 == 0:
        if method_flag:
            method.model = model
            method.model_weight_update()
            model = method.model
        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % args.print_freq == 0:
            print_log('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                  'Prec@5 {top5.val:.3f} ({top5.avg:.3f})\t''lambda: {s_e:.5f}'.format(
                epoch, i, len(train_loader), batch_time=batch_time,
                data_time=data_time, loss=losses, top1=top1, top5=top5, s_e=s_e), log)
    # method.print_index()

    return losses.avg

def validate(val_loader, model, criterion):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    with torch.no_grad():  # operations inside don't track history
        for i, (data, target) in enumerate(val_loader):
            if args.cuda:
                data, target = data.cuda(), target.cuda()

            # compute output
            output = model(data)
            loss = criterion(output, target)

            # measure accuracy and record loss
            prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
            losses.update(loss.item(), data.size(0))
            top1.update(prec1[0], data.size(0))
            top5.update(prec5[0], data.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            if i % args.print_freq == 0:
                print_log('Test: [{0}/{1}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                      'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                    i, len(val_loader), batch_time=batch_time, loss=losses,
                    top1=top1, top5=top5), log)

    print_log(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'
          .format(top1=top1, top5=top5), log)

    return top1.avg

def save_checkpoint(state, save_name, is_best):
    torch.save(state, save_name + '_model.pth.tar')
    if is_best:
        shutil.copyfile(save_name + '_model.pth.tar', save_name + '_best.pth.tar')

def print_log(print_string, log):
    print("{:}".format(print_string))
    log.write('{:}\n'.format(print_string))
    log.flush()

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(optimizer, epoch):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    if args.dataset == 'ILSVRC2012':
        epoch_step = [30, 60, 90]
    else:
        # epoch_step = [60, 120, 160]
        epoch_step = [0.5*args.epochs, 0.75*args.epochs]
        # epoch_step = [120, 160]
    if epoch in epoch_step:
        # args.lr *= 0.2
        args.lr *= 0.1
    for param_group in optimizer.param_groups:
        param_group['lr'] = args.lr

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

def sparse_coefficent(epoch, args):
    if args.dataset == 'ILSVRC2012':
        epoch_step = [30, 60, 90]
    else:
        # epoch_step = [60, 120, 160]
        epoch_step = [0.5*args.epochs, 0.75*args.epochs]
    s_e = 1.
    if epoch < epoch_step[0]:
        s_e = 1 / (1 + math.exp(15 - 30 * (epoch + 1) / epoch_step[0]))

    return s_e

class DPSS:
    def __init__(self, model, lad21, prt):
        self.model = model
        if args.arch == 'vggsmall':
            self.list1 = [0, 3, 7, 10, 14, 17, 20, 24, 27, 30, 34, 37, 40]
        elif args.arch == 'vggvariant':
            # self.list1 = [0, 3, 7, 10, 14, 17, 20, 23, 27, 30, 33, 36, 40, 43, 46, 49]
            self.list1 = [0, 3, 7, 10, 14, 17, 20, 24, 27, 30, 34, 37, 40]
        else:
            if args.dataset == 'ILSVRC2012':
                self.list1 = ['layer1', 'layer2', 'layer3', 'layer4']
            else:
                self.list1 = ['layer1', 'layer2', 'layer3']
        self.lambda_L21 = lad21
        self.prune_ratio = prt
        self.scale_coe = 1.
        self.s_c_v = 1.
        self.prune_thr = 1e-2
        self.layer_sparsity_ratio = []
        self.layer_sparsity_allocation_ratio = []

    def adjust_scale_coe(self, scale_coe):
        self.scale_coe = scale_coe

    def sparse_coefficent_value(self, value, stsr_flag):
        if not stsr_flag:
            self.s_c_v = value

    def get_update_index(self, conv_layer, next_conv_layer):
        weight_copy = conv_layer.weight.data.clone()
        weight_copy_next = next_conv_layer.weight.data.clone()
        c_l21_norm = torch.norm(torch.norm(torch.norm(weight_copy.data, 2, 3), 2, 2), 2, 1)
        first_layer_index = torch.reshape(torch.nonzero(c_l21_norm.gt(torch.max(c_l21_norm) * self.prune_thr)), (-1,))
        num = min(int(first_layer_index.shape[0] * self.scale_coe), conv_layer.out_channels)
        # void zero
        num = max(num, 1)
        t = torch.mul(weight_copy, conv_layer.weight.grad)
        # t = torch.where(t > 0., torch.zeros_like(t).cuda(), t)
        # print(t)
        t1 = torch.mul(weight_copy_next, next_conv_layer.weight.grad)
        # t1 = torch.where(t1 > 0., torch.zeros_like(t1).cuda(), t1)
        # print(t1)
        t_sum = torch.sum(t, (3, 2, 1))
        if isinstance(next_conv_layer, nn.Conv2d):
            t1_sum = torch.sum(t1, (3, 2, 0))
        else:
            t1_sum = torch.sum(t1, 0)
        # mul_t_t1 = torch.mul(t_l2, t1_l2)
        add_t_t1 = torch.abs(torch.add(t_sum, t1_sum))
        # print(t_l2)
        # print(t1_l2)
        # print(mul_t_t1)
        # print(add_t_t1)
        _, index = torch.topk(add_t_t1, conv_layer.out_channels - num, largest=False)
        return index, t_sum, t1_sum
    def print_index(self):
        if 'vgg' in args.arch:
            for i, layer_index in enumerate(self.list1):
                if i < len(self.list1) - 1:
                    print('feature: {0}\n'.format(layer_index), self.get_update_index(self.model.module._modules['feature'][layer_index], self.model.module._modules['feature'][self.list1[i + 1]])[1])
                    print('feature: {0}\n'.format(self.list1[i + 1]), self.get_update_index(self.model.module._modules['feature'][layer_index], self.model.module._modules['feature'][self.list1[i + 1]])[2])
                else:
                    print('feature: {0}\n'.format(layer_index), self.get_update_index(self.model.module._modules['feature'][layer_index], self.model.module._modules['classifier1'])[1])
                    print('feature: classifier1\n',
                          self.get_update_index(self.model.module._modules['feature'][layer_index],
                                                self.model.module._modules['classifier1'])[2])
        else:
            for layer_index, layer_index_value in enumerate(self.list1):
                for j in range(self.model.module.layernum[layer_index]):
                    print('{}''{:}.conv1\n'.format(layer_index_value, j), self.get_update_index(self.model.module._modules[layer_index_value][j].conv1, self.model.module._modules[layer_index_value][j].conv2)[1])
                    print('{}''{:}.conv2\n'.format(layer_index_value, j),
                          self.get_update_index(self.model.module._modules[layer_index_value][j].conv1,
                                                self.model.module._modules[layer_index_value][j].conv2)[2])
                    if args.arch in ['resnet50', 'resnet101']:
                        print('{}''{:}.conv2\n'.format(layer_index_value, j),
                              self.get_update_index(self.model.module._modules[layer_index_value][j].conv2,
                                                    self.model.module._modules[layer_index_value][j].conv3)[1])
                        print('{}''{:}.conv3\n'.format(layer_index_value, j),
                              self.get_update_index(self.model.module._modules[layer_index_value][j].conv2,
                                                    self.model.module._modules[layer_index_value][j].conv3)[2])

    def L21_regularizer(self, conv_layer, index):
        weight_copy = conv_layer.weight.data[index].clone()
        if weight_copy.shape[0] > 0:
            c_l21_norm = torch.norm(torch.norm(torch.norm(weight_copy.data, 2, 3), 2, 2), 2, 1)
            conv_layer.weight.data[index] = torch.reshape(torch.reshape(weight_copy, (weight_copy.shape[0], -1)).t().mul(
            torch.where(c_l21_norm > args.lr * self.lambda_L21 * self.s_c_v, torch.div(c_l21_norm - args.lr * self.lambda_L21 * self.s_c_v, c_l21_norm),
                        torch.zeros(c_l21_norm.shape).cuda())).t(), weight_copy.shape)
        return conv_layer.weight.data

    def L21_next_regularizer(self, conv_layer, index):
        weight_copy = conv_layer.weight.data[:, index].clone()
        if weight_copy.shape[1] > 0:
            if isinstance(conv_layer, nn.Conv2d):
                c_l21_norm = torch.norm(torch.norm(torch.norm(weight_copy.data, 2, 3), 2, 2), 2, 0)
            else:
                c_l21_norm = torch.norm(weight_copy.data, 2, 0)
            conv_layer.weight.data[:, index] = torch.transpose(torch.reshape(torch.reshape(torch.transpose(weight_copy, 0, 1), (weight_copy.shape[1], -1)).t().mul(
            torch.where(c_l21_norm > args.lr * self.lambda_L21 * self.s_c_v, torch.div(c_l21_norm - args.lr * self.lambda_L21 * self.s_c_v, c_l21_norm),
                        torch.zeros(c_l21_norm.shape).cuda())).t(), torch.transpose(weight_copy, 0, 1).shape), 0, 1)
        return conv_layer.weight.data

    def model_weight_update(self):
        if 'vgg' in args.arch:
            for i, layer_index in enumerate(self.list1):
                if self.lambda_L21 > 1e-6:
                    if i < len(self.list1) - 1:
                        update_index = self.get_update_index(self.model.module._modules['feature'][layer_index], self.model.module._modules['feature'][self.list1[i + 1]])[0]
                        self.model.module._modules['feature'][layer_index].weight.data = self.L21_regularizer(self.model.module._modules['feature'][layer_index], update_index)
                        self.model.module._modules['feature'][self.list1[i + 1]].weight.data = self.L21_next_regularizer(
                            self.model.module._modules['feature'][self.list1[i + 1]], update_index)
                    else:
                        update_index = self.get_update_index(self.model.module._modules['feature'][layer_index], self.model.module._modules['classifier1'])[0]
                        self.model.module._modules['feature'][layer_index].weight.data = self.L21_regularizer(self.model.module._modules['feature'][layer_index], update_index)
                        self.model.module._modules['classifier1'].weight.data = self.L21_next_regularizer(
                            self.model.module._modules['classifier1'], update_index)

        else:
            for layer_index, layer_index_value in enumerate(self.list1):
                for j in range(self.model.module.layernum[layer_index]):
                    if self.lambda_L21 > 1e-6:
                        update_index = self.get_update_index(self.model.module._modules[layer_index_value][j].conv1, self.model.module._modules[layer_index_value][j].conv2)[0]
                        self.model.module._modules[layer_index_value][j].conv1.weight.data = self.L21_regularizer(self.model.module._modules[layer_index_value][j].conv1, update_index)
                        self.model.module._modules[layer_index_value][j].conv2.weight.data = self.L21_next_regularizer(
                            self.model.module._modules[layer_index_value][j].conv2, update_index)
                    if args.arch in ['resnet50', 'resnet101']:
                        if self.lambda_L21 > 1e-6:
                            update_index = self.get_update_index(self.model.module._modules[layer_index_value][j].conv2, self.model.module._modules[layer_index_value][j].conv3)[0]
                            self.model.module._modules[layer_index_value][j].conv2.weight.data = self.L21_regularizer(self.model.module._modules[layer_index_value][j].conv2, update_index)
                            self.model.module._modules[layer_index_value][j].conv3.weight.data = self.L21_next_regularizer(
                                self.model.module._modules[layer_index_value][j].conv3, update_index)

    def row_prune(self, layer_p, BN_layer_p, layer1_p, pretrained=False):
        c_l21_norm = torch.norm(torch.norm(torch.norm(layer_p.weight.data, 2, 3), 2, 2), 2, 1)
        if isinstance(layer1_p, nn.Conv2d):
            c_l21_norm += torch.norm(torch.norm(torch.norm(layer1_p.weight.data, 2, 3), 2, 2), 2, 0)
        else:
            c_l21_norm += torch.norm(layer1_p.weight.data, 2, 0)
        first_layer_index = torch.reshape(torch.nonzero(c_l21_norm.gt(torch.max(c_l21_norm) * self.prune_thr)), (-1,))
        self.layer_sparsity_ratio.append(1.-first_layer_index.shape[0]/c_l21_norm.shape[0])
        num = min(int(first_layer_index.shape[0] * self.scale_coe), layer_p.out_channels)
        num = max(num, 1)
        self.layer_sparsity_allocation_ratio.append(1. - num / layer_p.out_channels)
        _, first_layer_index = torch.topk(c_l21_norm, num, largest=True)
        # l21_norm = torch.norm(torch.norm(torch.norm(layer_p.weight.data, 2, 1), 2, 1), 2, 1)
        # tk = int(layer_p.weight.data.shape[0] * (1. - self.prune_ratio))
        # _, first_layer_index = torch.topk(l21_norm, tk, largest=True)
        first_layer_p = nn.Conv2d(in_channels=layer_p.in_channels, out_channels=first_layer_index.shape[0], kernel_size=layer_p.kernel_size, \
                               stride=layer_p.stride, padding=layer_p.padding, bias=layer_p.bias)
        first_layer_p.weight.data = layer_p.weight.data[first_layer_index]
        if isinstance(layer1_p, nn.Conv2d):
            last_layer_p = nn.Conv2d(in_channels=first_layer_index.shape[0], \
                                out_channels=layer1_p.out_channels, kernel_size=layer1_p.kernel_size, stride=layer1_p.stride, padding=layer1_p.padding, bias=layer1_p.bias)
            last_layer_p.weight.data = layer1_p.weight.data[:,first_layer_index]
        elif isinstance(layer1_p, nn.Linear):
            last_layer_p = nn.Linear(first_layer_index.shape[0], layer1_p.out_features)
            last_layer_p.weight.data = layer1_p.weight.data[:,first_layer_index]
            last_layer_p.bias.data = layer1_p.bias.data
        bn_layer_p = nn.BatchNorm2d(first_layer_index.shape[0])
        bn_layer_p.weight.data = BN_layer_p.weight.data[first_layer_index].clone()
        bn_layer_p.bias.data = BN_layer_p.bias.data[first_layer_index].clone()
        bn_layer_p.running_mean = BN_layer_p.running_mean[first_layer_index].clone()
        bn_layer_p.running_var = BN_layer_p.running_var[first_layer_index].clone()
        return first_layer_p, bn_layer_p, last_layer_p

    def channel_prune(self):
        if 'vgg' in args.arch:
            for i, layer_index in enumerate(self.list1):
                if i < len(self.list1) - 1:
                    self.model.module._modules['feature'][layer_index], self.model.module._modules['feature'][layer_index + 1], self.model.module._modules['feature'][self.list1[i + 1]] = \
                self.row_prune(self.model.module._modules['feature'][layer_index], self.model.module._modules['feature'][layer_index + 1], self.model.module._modules['feature'][self.list1[i + 1]])
                else:
                    self.model.module._modules['feature'][layer_index], self.model.module._modules['feature'][layer_index + 1], self.model.module._modules['classifier1'] = \
                    self.row_prune(self.model.module._modules['feature'][layer_index], self.model.module._modules['feature'][layer_index + 1], self.model.module._modules['classifier1'])
        else:
            for layer_index, layer_index_value in enumerate(self.list1):
                for j in range(self.model.module.layernum[layer_index]):
                    self.model.module._modules[layer_index_value][j].conv1, \
                    self.model.module._modules[layer_index_value][j].bn1, \
                    self.model.module._modules[layer_index_value][j].conv2 = self.row_prune(
                        self.model.module._modules[layer_index_value][j].conv1,
                        self.model.module._modules[layer_index_value][j].bn1,
                        self.model.module._modules[layer_index_value][j].conv2)
                    if args.arch in ['resnet50', 'resnet101']:
                        self.model.module._modules[layer_index_value][j].conv2, \
                        self.model.module._modules[layer_index_value][j].bn2, \
                        self.model.module._modules[layer_index_value][j].conv3 = self.row_prune(
                            self.model.module._modules[layer_index_value][j].conv2,
                            self.model.module._modules[layer_index_value][j].bn2,
                            self.model.module._modules[layer_index_value][j].conv3)

if __name__ == '__main__':
    main()


import messages
import os.path as pt
import shutil
import sys
import time
import tempfile
import libtorrent as lt

def torrent_params(tempdir):
    return {
        'save_path': tempdir,
        'storage_mode': lt.storage_mode_t(2),
        'paused': False,
        'auto_managed': True,
        'duplicate_is_error': True
    }

def wait_for_metadata(handle, ses, tempdir):
    while not handle.has_metadata():
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            ses.pause()
            shutil.rmtree(tempdir)
            sys.exit(0)
    ses.pause()
    print("Metadata finished")

def output_file(torinfo, tmp_file):
    output = pt.abspath(torinfo.name() + ".torrent")
    if tmp_file:
        if pt.isdir(tmp_file):
            return pt.abspath(pt.join(tmp_file, torinfo.name() + ".torrent"))
        elif pt.isdir(pt.dirname(pt.abspath(tmp_file))):
            return pt.abspath(tmp_file)

def convert(link, tmp_file):
    tempdir = tempfile.mkdtemp()
    ses = lt.session()
    params = torrent_params(tempdir)
    handle = lt.add_magnet_uri(ses, link, params)

    print("Downloading Metadata (this may take a while)")
    wait_for_metadata(handle, ses, tempdir)

    torinfo = handle.get_torrent_info()
    torfile = lt.create_torrent(torinfo)
    output = output_file(torinfo, tmp_file)

    print("Saving torrent file here " + output + " ...")
    torcontent = lt.bencode(torfile.generate())
    f = open(output, "wb")
    f.write(lt.bencode(torfile.generate()))
    f.close()
    print("Saved! Cleaning up dir: " + tempdir)
    ses.remove_torrent(handle)
    shutil.rmtree(tempdir)
    return output