import os
import sys
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geonode.settings")
if __name__ == '__main__':
    import django
    django.setup()

import urllib3
from bs4 import BeautifulSoup
import datetime

from geonode.harvester.utils import create_thumbnail, save_document

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

url = 'https://www.ctc-n.org/type-resource/document'
total_added = 0
harvester_id = 1000  # admin user id
datasource = 'ctc-n.org/type-resource/document'

def get_records(url):
    http = urllib3.PoolManager()
    r = http.request("GET", url)
    return BeautifulSoup(r.data, 'lxml')

def main(url={}):
    global total_added

    getsoup = get_records(url=url)
    pages = getsoup.find("li", class_='pager-last last').a['href']
    # total = pages.split('=')[1]
    total = 10

    while (total_added < int(total)):
        url = 'https://www.ctc-n.org/type-resource/document?page='+str(total_added)
        print("Web Page: ", url)
        total_added = total_added + 1

        getpost = get_records(url=url)
        content = getpost.find('section', {"id":"block-system-main"})
        posts = content.find('div', class_="inner").find_all('div', class_='node node-resource node-teaser clearfix')

        for post in posts:
            geturl = post.find('h2', class_="teaser-title").a['href']
            posturl = 'https://www.ctc-n.org'+geturl

            image = post.find("div", class_="field-name-field-document")
            checkimage = None if image == None else image.find(
                'div', class_="field-item even").find('img', class_="pdfpreview-file")
            getimage = None if checkimage == None else checkimage["src"]
            pdf = post.find("div", class_="field-name-field-document").find('div', class_="field-item even").find('span', class_="file")

            if checkimage == None:
                if pdf is not None:
                    getpdfurl = pdf.a['href']
                    getfileicon = pdf.find('img', class_="file-icon")["src"]
            else:
                getfileicon = getimage
                getpdfurl = None
            print("Web Post: ", posturl)
            harvest_all(posturl, getfileicon)

    print("Added ", total_added)

def harvest_all(url, file, db=None, **kwargs):
    global total_added

    soup = get_records(url)
    results = soup.find_all("div", class_="subcontainer")

    for result in results:
        title = result.find('h1', class_="page-header").text

        content = result.find('section', {"id":"block-system-main"}).find('div', class_="inner").find('div', class_="node node-resource clearfix").find('div', class_="content")
        postdate = content.find("div", class_="field-name-field-publication-date")
        body = content.find("div", class_="field-name-body")

        getpostdate = None if postdate == None else postdate.find('div', class_="field-item even").span['content']
        getbody = None if body == None else body.find('div', property="content:encoded").text

        docparams = {
            'title': title,
            'owner_id': harvester_id,
            'doc_url': url,
            'datasource': datasource,
            'date': getpostdate,
            'abstract': getbody,
        }
        specialparams = {
            'external_thumbnail_url': file,
        }

        save_mode = save_document(docparams, specialparams, insertonly=kwargs.get('insertonly') or kwargs.get('insertnewonly'))

        create_thumbnail(
            doc_url=docparams['doc_url'],
            doc=None,
            external_thumbnail_url=specialparams['external_thumbnail_url']
        )

    return soup

if __name__ == "__main__":
    main(url)
    # pass


# -*- coding: utf-8 -*-
"""snaProjectRound-I.ipynb

**Importing the required Libraries**
"""

import numpy as np
import pandas as pd
import networkx as nx
import matplotlib.pyplot as plt
from math import log

"""**Dataset Download**"""

urlFacebookNetwork = 'https://snap.stanford.edu/data/facebook_combined.txt.gz'
urlEmailNetwork = 'https://snap.stanford.edu/data/email-Eu-core.txt.gz'

dataframeFacebookNetwork = pd.read_csv(urlFacebookNetwork, delimiter=' ', header=None, names=['from', 'to'])
dataframeEmailNetwork = pd.read_csv(urlEmailNetwork, delimiter=' ', header=None, names=['from', 'to'])

facebookNetwork = nx.from_pandas_edgelist(dataframeFacebookNetwork, 'from', 'to')
facebookNetwork.name = 'Facebook Network'
emailNetwork = nx.from_pandas_edgelist(dataframeEmailNetwork, 'from', 'to', create_using=nx.DiGraph())
emailNetwork.name = 'E-mail Network'

"""**Information about the Networks**"""

print(nx.info(facebookNetwork))
print()
print(nx.info(emailNetwork))

"""**Networks**

Facebook Network
"""

plt.figure(figsize=(50, 50))
plt.title('Facebook Network', fontsize=20)
nx.draw_networkx(facebookNetwork, pos=nx.spring_layout(facebookNetwork))

"""Email Network"""

plt.figure(figsize=(50, 50))
plt.title('Email Network', fontsize=20)
nx.draw_networkx(emailNetwork, pos=nx.spring_layout(emailNetwork))

"""**Centrality Measures**

Degree Centrality
"""

degCentralityFacebookNetwork = nx.degree_centrality(facebookNetwork)
pd.DataFrame(sorted(degCentralityFacebookNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'degreeCentrality'])

inDegCentralityEmailNetwork = nx.in_degree_centrality(emailNetwork)
pd.DataFrame(sorted(inDegCentralityEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'inDegreeCentrality'])

outDegCentralityEmailNetwork = nx.out_degree_centrality(emailNetwork)
pd.DataFrame(sorted(outDegCentralityEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'outDegreeCentrality'])

"""EigenVector Centrality"""

evCentralityFaceBookNetwork = nx.eigenvector_centrality(facebookNetwork)
pd.DataFrame(sorted(evCentralityFaceBookNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'eigenvectorCentrality'])

evCentralityEmailNetwork = nx.eigenvector_centrality(emailNetwork)
pd.DataFrame(sorted(evCentralityEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'eigenvectorCentrality'])

"""Katz Centrality"""

katzCentralityFacebookNetwork = nx.katz_centrality_numpy(facebookNetwork)
pd.DataFrame(sorted(katzCentralityFacebookNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'katzCentrality'])

katzCentralityEmailNetwork = nx.katz_centrality_numpy(emailNetwork)
pd.DataFrame(sorted(katzCentralityEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'katzCentrality'])

"""PageRank Centrality"""

pagerankCentralityFacebookNetwork = nx.pagerank(facebookNetwork)
pd.DataFrame(sorted(pagerankCentralityFacebookNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'pagerankCentrality'])

pagerankCentralityEmailNetwork = nx.pagerank(emailNetwork)
pd.DataFrame(sorted(pagerankCentralityEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'pagerankCentrality'])

"""Closeness Centrality"""

closenessCentralityFacebookNetwork = nx.closeness_centrality(facebookNetwork)
pd.DataFrame(sorted(closenessCentralityFacebookNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'closenessCentrality'])

closenessCentralityEmailNetwork = nx.closeness_centrality(emailNetwork)
pd.DataFrame(sorted(closenessCentralityEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'closenessCentrality'])

"""Betweenness Centrality"""

betweennessCentralityFacebookNetwork = nx.betweenness_centrality(facebookNetwork)
pd.DataFrame(sorted(betweennessCentralityFacebookNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'betweennessCentrality'])

betweennessCentralityEmailNetwork = nx.betweenness_centrality(emailNetwork)
pd.DataFrame(sorted(betweennessCentralityEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'betweennessCentrality'])

"""**Local Clustering Coefficients**"""

localClusteringCoefficientsFacebookNetwork = nx.clustering(facebookNetwork)
pd.DataFrame(sorted(localClusteringCoefficientsFacebookNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'localClusteringCoefficients'])

localClusteringCoefficientsEmailNetwork = nx.clustering(emailNetwork)
pd.DataFrame(sorted(localClusteringCoefficientsEmailNetwork.items(), key=lambda item: item[1], reverse=True), columns=['Node', 'localClusteringCoefficients'])

"""**Average Clustering Coefficient**"""

averageClusteringCoefficientFacebookNetwork = nx.average_clustering(facebookNetwork)
averageClusteringCoefficientFacebookNetwork

averageClusteringCoefficientEmailNetwork = nx.average_clustering(emailNetwork)
averageClusteringCoefficientEmailNetwork

"""**Transitivity/ Global Clustering Coefficient**"""

transitivityFacebookNetwork = nx.transitivity(facebookNetwork)
transitivityFacebookNetwork

transitivityEmailNetwork = nx.transitivity(emailNetwork)
transitivityEmailNetwork

"""**Reciprocity**"""

reciprocityFacebookNetwork = nx.reciprocity(facebookNetwork)
reciprocityFacebookNetwork

reciprocityEmailNetwork = nx.reciprocity(emailNetwork)
reciprocityEmailNetwork

"""**Giant Components**

Giant Component of Facebook Network
"""

plt.figure(figsize=(50, 50))
plt.title('Giant Component of Facebook Network', fontsize=20)
giantFacebookNetwork = facebookNetwork.subgraph(max(nx.connected_components(facebookNetwork), key=len))
nx.draw_networkx(facebookNetwork, pos=nx.spring_layout(facebookNetwork), edgelist=giantFacebookNetwork.edges())

giantFacebookNetwork.number_of_nodes()

"""Giant Component of Email Network"""

plt.figure(figsize=(50, 50))
plt.title('Giant Component of Email Network', fontsize=20)
giantEmailNetwork = emailNetwork.subgraph(max(nx.strongly_connected_components(emailNetwork), key=len))
nx.draw_networkx(emailNetwork, pos=nx.spring_layout(emailNetwork), edgelist=giantEmailNetwork.edges())

giantEmailNetwork.number_of_nodes()

"""**Evolution of Random Network**"""

plt.figure(figsize=(30,10))
plt.title('Evolution of Random Network', fontsize=20)
nodes = 10000
averageDegree = np.arange(0, 10.1, 0.1)
probabilityK = []
for degree in averageDegree:
  randomNetwork = nx.erdos_renyi_graph(nodes, degree/(nodes-1))
  giantRandomNetwork = randomNetwork.subgraph(max(nx.connected_components(randomNetwork), key=len))
  probabilityK.append(giantRandomNetwork.number_of_nodes()/nodes)
plt.plot(averageDegree, probabilityK, 'm', linewidth=5)
plt.axvline(x=1, color='k', linewidth=2)
plt.axvline(x=log(nodes), color='k', linewidth=2)
plt.xlabel('<k>')
plt.ylabel('N_G/N')
plt.show()


import datetime
from io import BytesIO

from django.conf import settings
from django.contrib.auth.decorators import permission_required
from django.core.exceptions import ValidationError
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import render, redirect
import django.views.generic as views
from django.template.defaultfilters import slugify
from django.urls import reverse_lazy, reverse
from django.contrib.auth import mixins as auth_mixins

from service_manager.core.mixins import MaterialFilteringMixin
from service_manager.core.utils import get_protocol_and_domain_as_string, get_material_count_matching_conditions
from service_manager.main.forms import CreateServiceOrderHeaderForm, CreateServiceOrderDetailForm, \
    EditServiceOrderDetailForm, CreateServiceOrderNoteForm, HandoverServiceOrderForm, ContactForm, TrackOrderSearchForm, \
    CreateCustomerNotificationForm
from service_manager.main.models import Customer, CustomerAsset, ServiceOrderHeader, ServiceOrderDetail, \
    ServiceOrderNote, CustomerNotification, ServiceRequest
from service_manager.main.tasks import send_contact_us_email
from django.contrib import messages
from django.utils.translation import gettext_lazy as _
from weasyprint import HTML, CSS
import qrcode
import qrcode.image.svg

from service_manager.master_data.forms import FilterMaterialForm
from service_manager.master_data.models import Material

class HomeTemplateView(views.TemplateView):
    template_name = 'index.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        # Service requests
        open_service_requests = ServiceRequest.objects.all().filter(status__in=[1, 2, 3]).order_by('-created_on')
        open_service_requests_count = open_service_requests.count()
        open_service_requests = open_service_requests[:3]

        # Pending service orders
        pending_service_orders = ServiceOrderHeader.objects.all().filter(is_serviced=False).order_by('-created_on')
        pending_service_orders_count = pending_service_orders.count()
        pending_service_orders = pending_service_orders[:3]

        # Completed service orders
        completed_service_orders = ServiceOrderHeader.objects.all().filter(
            Q(is_serviced=True) & Q(is_completed=False)).order_by('-created_on')
        completed_service_orders_count = completed_service_orders.count()
        completed_service_orders = completed_service_orders[:3]

        context['open_service_requests'] = open_service_requests
        context['open_service_requests_count'] = open_service_requests_count

        context['pending_service_orders'] = pending_service_orders
        context['pending_service_orders_count'] = pending_service_orders_count

        context['completed_service_orders'] = completed_service_orders
        context['completed_service_orders_count'] = completed_service_orders_count

        return context

class ServiceOrderHeaderPendingServiceListView(auth_mixins.PermissionRequiredMixin, views.ListView):
    model = ServiceOrderHeader
    template_name = 'service_order_header/list_views/service_orders_service.html'
    ordering = (
        'service_level_agreement', 'customer__has_subscription', 'customer__is_regular_customer', 'created_on',
        'customer')
    RELATED_ENTITIES = ['serviceordernote_set', 'serviceorderdetail_set', 'customer']

    permission_required = 'main.view_serviceorderheader'

    def get_context_data(self, *, object_list=None, **kwargs):
        context = super().get_context_data(**kwargs)
        queryset = self.get_queryset()

        sla_orders = queryset.filter(service_level_agreement=True)

        subscription_customers = queryset.filter(customer__has_subscription=True).exclude(
            id__in=sla_orders.values_list('id', flat=True)
        )

        regular_customers = queryset.filter(customer__is_regular_customer=True).exclude(
            Q(id__in=sla_orders.values_list('id', flat=True)) |
            Q(id__in=subscription_customers.values_list('id', flat=True))
        )

        others = queryset.exclude(
            Q(id__in=sla_orders.values_list('id', flat=True)) |
            Q(id__in=subscription_customers.values_list('id', flat=True)) |
            Q(id__in=regular_customers.values_list('id', flat=True))
        )

        context['sla_orders'] = sla_orders
        context['subscription_customers'] = subscription_customers
        context['regular_customers'] = regular_customers
        context['others'] = others

        return context

    def get_queryset(self):
        queryset = super().get_queryset()
        return queryset.filter(is_serviced=False).prefetch_related(*self.RELATED_ENTITIES)

class ServiceOrderHeaderServicedListView(auth_mixins.PermissionRequiredMixin, views.ListView):
    model = ServiceOrderHeader
    template_name = 'service_order_header/list_views/service_orders_complete.html'
    ordering = ('serviced_on', 'customer')
    RELATED_ENTITIES = ['serviceordernote_set', 'serviceorderdetail_set', ]

    permission_required = 'main.view_serviceorderheader'

    def get_queryset(self):
        queryset = super().get_queryset()
        return queryset.filter(Q(is_serviced=True) & Q(is_completed=False)).prefetch_related(*self.RELATED_ENTITIES)

class ServiceOrderHeaderDetailView(auth_mixins.PermissionRequiredMixin, views.DetailView):
    model = ServiceOrderHeader
    template_name = 'service_order_header/core/service_order_details.html'
    context_object_name = 'service_order_header'

    permission_required = 'main.view_serviceorderheader'

    # Override the queryset in order to include the (soft) deleted SOHs
    # This is needed when showing the service history of a given Customer Asset
    def get_queryset(self, *args, **kwargs):
        queryset = ServiceOrderHeader.all_records.all()
        return queryset

class CreateServiceOrderHeader(auth_mixins.PermissionRequiredMixin, views.CreateView):
    model = ServiceOrderHeader
    form_class = CreateServiceOrderHeaderForm
    template_name = 'service_order_header/core/service_order_create.html'

    permission_required = 'main.add_serviceorderheader'

    def get(self, request, *args, **kwargs):

        if 'service_request_id' in self.request.GET:
            self.kwargs['service_request_id'] = self.request.GET['service_request_id']

        return super().get(request, *args, **kwargs)

    def get_success_url(self):
        return reverse_lazy(
            'create_service_order_success',
            kwargs=
            {
                'pk': self.object.id,
            }
        )

    def get_initial(self):
        customer_id = self.kwargs['customer_id']
        if customer_id:
            self.initial.update({
                'customer': customer_id,
            })

        return super().get_initial()

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        customer_id = self.kwargs['customer_id']
        customer_asset_id = self.kwargs['asset_id']

        if customer_id:
            customer = Customer.objects.get(pk=customer_id)
            context['customer'] = customer
        if customer_asset_id:
            customer_asset = CustomerAsset.objects.get(pk=customer_asset_id)
            context['customer_asset'] = customer_asset

        return context

    def form_valid(self, form):
        customer_id = self.kwargs['customer_id']
        customer_asset_id = self.kwargs['asset_id']

        service_order = form.save(commit=False)
        service_order.accepted_by = self.request.user
        service_order.customer = Customer.objects.get(pk=customer_id)
        service_order.customer_asset = CustomerAsset.objects.get(pk=customer_asset_id)

        # Update the Service request if applicable
        if 'service_request_id' in self.request.GET:
            service_request_id = self.request.GET['service_request_id']
            service_request = ServiceRequest.objects.get(pk=service_request_id)
            # todo: make this exception better
            if service_request.customer != service_order.customer:
                raise ValidationError('Service request customer is different from the Service order customer!')
            service_request.service_order = service_order
            service_order.save()
            service_request.save()
        else:
            service_order.save()

        # Create a slug
        created_on = service_order.created_on
        slug = f'{hex(service_order.pk)}-{hex(created_on.year)}-{hex(created_on.month)}-{hex(created_on.day)}'
        service_order.slug = slugify(slug)

        return super().form_valid(form)

class CreateServiceOrderSuccess(auth_mixins.PermissionRequiredMixin, views.DetailView):
    model = ServiceOrderHeader
    context_object_name = 'service_order'
    template_name = 'service_order_header/create_service_order_header_success.html'
    permission_required = 'main.view_serviceorderheader'

class DeleteServiceOrderHeaderView(auth_mixins.PermissionRequiredMixin, views.DeleteView):
    model = ServiceOrderHeader
    template_name = 'service_order_header/core/service_order_delete.html'
    success_url = reverse_lazy('service_orders_list_pending_service')

    permission_required = 'main.change_serviceorderheader'

class CreateServiceOrderDetailView(MaterialFilteringMixin, auth_mixins.PermissionRequiredMixin, views.CreateView):
    model = ServiceOrderDetail
    template_name = 'service_order_detail/service_order_details_create.html'
    form_class = CreateServiceOrderDetailForm

    permission_required = 'main.add_serviceorderdetail'

    def get_success_url(self):
        service_order_header_id = self.kwargs['order_id']
        return reverse_lazy('create_service_order_detail', kwargs={'order_id': service_order_header_id})

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        context['filter_form'] = FilterMaterialForm(self.request.GET or None)

        service_order_header_id = self.kwargs['order_id']
        if service_order_header_id:
            context['service_order_header'] = ServiceOrderHeader.objects.prefetch_related(
                'serviceorderdetail_set').get(pk=int(service_order_header_id))

        context['material_count'] = get_material_count_matching_conditions(
            self.request.GET.get('category' or None),
            self.request.GET.get('search' or None)
        )

        return context

    def form_valid(self, form):
        service_order_header_id = self.kwargs['order_id']
        service_order_header = ServiceOrderHeader.objects.get(pk=service_order_header_id)
        service_order_detail = form.save(commit=False)

        service_order_detail.service_order = service_order_header
        service_order_detail.save()

        return super().form_valid(form)

class EditServiceOrderDetailView(auth_mixins.PermissionRequiredMixin, views.UpdateView):
    model = ServiceOrderDetail
    template_name = 'service_order_detail/service_order_detail_edit.html'
    form_class = EditServiceOrderDetailForm

    permission_required = 'main.change_serviceorderdetail'

    def get_success_url(self):
        service_order_header_id = self.kwargs['order_id']
        if service_order_header_id:
            return reverse_lazy('create_service_order_detail', kwargs={'order_id': service_order_header_id})
        return reverse_lazy('service_orders_list')

class ServiceOrderDetailsListView(auth_mixins.PermissionRequiredMixin, views.ListView):
    model = ServiceOrderDetail
    template_name = 'service_order_detail/service_order_details.html'
    ordering = ('material', 'quantity',)

    permission_required = 'main.view_serviceorderdetail'

class DeleteServiceOrderDetailView(auth_mixins.PermissionRequiredMixin, views.DeleteView):
    model = ServiceOrderDetail
    template_name = 'service_order_detail/service_order_detail_delete.html'

    permission_required = 'main.change_serviceorderdetail'

    def get_success_url(self):
        service_order_header_id = self.kwargs['order_id']
        if service_order_header_id:
            return reverse_lazy('create_service_order_detail', kwargs={'order_id': service_order_header_id})
        return reverse_lazy('service_orders_list')

@permission_required('main.change_serviceorderheader')
def complete_service_order(request, pk):
    service_order_header = ServiceOrderHeader.objects.get(pk=pk)
    service_order_header.is_serviced = True
    service_order_header.serviced_by = request.user
    service_order_header.serviced_on = datetime.datetime.now()
    service_order_header.save()

    return redirect('service_orders_list_pending_service')

class CreateServiceOrderNoteView(auth_mixins.PermissionRequiredMixin, views.CreateView):
    model = ServiceOrderNote
    template_name = 'service_order_note/service_order_note_create.html'
    form_class = CreateServiceOrderNoteForm

    permission_required = 'main.add_serviceordernote'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        service_order_id = self.kwargs['order_id']
        if service_order_id:
            service_order = ServiceOrderHeader.objects.filter(pk=service_order_id).get()
            context['service_order'] = service_order

        return context

    def get_success_url(self):
        go_to_next = self.request.POST.get('next', '/')
        return go_to_next

    def form_valid(self, form):
        service_order_id = self.kwargs['order_id']
        service_order_header = ServiceOrderHeader.objects.get(pk=service_order_id)

        note = form.save(commit=False)
        note.service_order = service_order_header
        note.created_by = self.request.user
        note.save()

        return super().form_valid(form)

class ServiceOrderNotesListView(auth_mixins.PermissionRequiredMixin, views.ListView):
    model = ServiceOrderNote
    template_name = 'service_order_note/service_order_notes.html'

    permission_required = 'main.view_serviceordernote'

class ServiceOrderNoteDetailView(auth_mixins.PermissionRequiredMixin, views.DetailView):
    model = ServiceOrderNote
    template_name = 'service_order_note/service_order_note_detail.html'

    permission_required = 'main.view_serviceordernote'

class EditServiceOrderNoteView(auth_mixins.PermissionRequiredMixin, views.UpdateView):
    model = ServiceOrderNote
    template_name = 'service_order_note/service_order_note_edit.html'
    form_class = CreateServiceOrderNoteForm

    permission_required = 'main.change_serviceordernote'

    def get_success_url(self):
        go_to_next = self.request.POST.get('next', '/')
        return go_to_next

class DeleteServiceOrderNoteView(auth_mixins.PermissionRequiredMixin, views.DeleteView):
    model = ServiceOrderNote
    template_name = 'service_order_note/service_order_note_delete.html'

    permission_required = 'main.change_serviceordernote'

    def get_success_url(self):
        go_to_next = self.request.POST.get('next', '/')
        return go_to_next

@permission_required('main.change_serviceorderheader')
def rollback_service_order(request, pk):
    service_order_header = ServiceOrderHeader.objects.get(pk=pk)
    service_order_header.is_serviced = False
    service_order_header.serviced_by = None
    service_order_header.serviced_on = None
    service_order_header.save()

    return redirect('service_orders_list_pending_service')

class HandoverServiceOrderView(auth_mixins.PermissionRequiredMixin, views.UpdateView):
    model = ServiceOrderHeader
    form_class = HandoverServiceOrderForm
    template_name = 'service_order_header/core/service_order_handover.html'
    # success_url = reverse_lazy('service_orders_list_pending_service')

    permission_required = 'main.change_serviceorderheader'

    def get_initial(self):
        service_order_header_id = self.kwargs['pk']

        if service_order_header_id:
            self.initial.update({
                'service_order': service_order_header_id,
            })

        return super().get_initial()

    def form_valid(self, form):
        service_oder = form.save(commit=False)

        service_oder.is_completed = True
        service_oder.completed_by = self.request.user
        service_oder.completed_on = datetime.datetime.now()
        service_oder.save()

        return super().form_valid(form)

    def get_success_url(self):
        order_id = self.kwargs['pk']
        return reverse_lazy('detail_service_order', kwargs={'pk': order_id})

def contact_us(request):
    if request.method == 'POST':
        form = ContactForm(request.POST)
        if form.is_valid():
            email_data = {
                'first_name': form.cleaned_data['first_name'],
                'last_name': form.cleaned_data['last_name'],
                'email_address': form.cleaned_data['email_address'],
                'message': form.cleaned_data['message'],
            }
            send_contact_us_email.delay(email_data)
            messages.success(request, _("The email message was sent."))
            return redirect('index')
        else:
            for key, error in list(form.errors.items()):
                if key == 'captcha' and error[0] == 'This field is required.':
                    messages.error(request, "You must pass the reCAPTCHA test")
                    continue
                messages.error(request, error)

    form = ContactForm()
    context = {
        'form': form,
    }
    return render(request, 'contact_us.html', context)

class TrackOrderDetailView(views.DetailView):
    model = ServiceOrderHeader
    template_name = 'service_order_header/service_order_track.html'
    context_object_name = 'order'

class ServiceOrderPrintoutView(views.DetailView):
    template_name = 'service_order_header/customer_printout.html'
    model = ServiceOrderHeader
    context_object_name = 'order'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        # Generate the link in the QR code
        domain = get_protocol_and_domain_as_string()
        code_text = f"{domain}{reverse('track_order', kwargs={'slug': kwargs['object'].slug})}"

        # Build the QR code
        factory = qrcode.image.svg.SvgImage
        img = qrcode.make(code_text, image_factory=factory, box_size=5)

        # Save as stream
        stream = BytesIO()
        img.save(stream)

        # Pass to the context
        context["qrcode"] = stream.getvalue().decode()
        return context

    def get(self, request, *args, **kwargs):
        template_response = super().get(self, request, *args, **kwargs)
        styles = CSS(settings.STATIC_ROOT + "/css/main.css")
        html = HTML(string=template_response.rendered_content).write_pdf(
            stylesheets=[
                styles,
            ]
        )

        response = HttpResponse(html, content_type='application/pdf')

        response['Content-Disposition'] = f'filename=Customer_Printout_{kwargs["pk"]}.pdf'
        return response

class TrackOrderSearchFormView(views.FormView):
    template_name = 'service_order_header/service_order_track_search.html'
    form_class = TrackOrderSearchForm

    def form_valid(self, form):
        order_slug = form.cleaned_data['order_tracking_number'].strip().lower()
        service_order = ServiceOrderHeader.objects.filter(slug__exact=order_slug)

        if not service_order:
            messages.error(self.request, _("Invalid Tracking number!"))
            return self.render_to_response(self.get_context_data(request=self.request, form=form))
        else:
            self.success_url = reverse('track_order', kwargs={'slug': order_slug})
            return super().form_valid(form)

    def form_invalid(self, form):
        for key, error in list(form.errors.items()):
            if key == 'captcha' and error[0] == 'This field is required.':
                messages.error(self.request, "You must pass the reCAPTCHA test")
                continue
            messages.error(self.request, error)
        return self.render_to_response(self.get_context_data(request=self.request, form=form))

class CustomerNotificationsListView(auth_mixins.PermissionRequiredMixin, views.ListView):
    model = CustomerNotification
    template_name = 'customer_notification/customer_notifications.html'

    # todo: create a new permission and change here and in the template as well
    permission_required = 'main.view_serviceordernote'

class CreateCustomerNotificationView(auth_mixins.PermissionRequiredMixin, views.CreateView):
    model = CustomerNotification
    template_name = 'customer_notification/customer_notification_create.html'
    form_class = CreateCustomerNotificationForm

    # todo: create a new permission and change here and in the template as well
    permission_required = 'main.add_serviceordernote'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        service_order_id = self.kwargs['order_id']
        if service_order_id:
            service_order = ServiceOrderHeader.objects.filter(pk=service_order_id).get()
            context['service_order'] = service_order

        return context

    def get_success_url(self):
        go_to_next = self.request.POST.get('next', '/')
        return go_to_next

    def form_valid(self, form):
        service_order_id = self.kwargs['order_id']
        service_order_header = ServiceOrderHeader.objects.get(pk=service_order_id)

        notification = form.save(commit=False)
        notification.service_order = service_order_header
        notification.notified_by = self.request.user
        notification.notification_method = CustomerNotification.TYPE_CHOICE_PHONE
        notification.service_order_current_status = service_order_header.status
        notification.save()

        return super().form_valid(form)

class EditCustomerNotificationView(auth_mixins.PermissionRequiredMixin, views.UpdateView):
    model = CustomerNotification
    template_name = 'customer_notification/customer_notification_edit.html'
    form_class = CreateCustomerNotificationForm

    permission_required = 'main.change_serviceordernote'

    def get_success_url(self):
        go_to_next = self.request.POST.get('next', '/')
        return go_to_next

class DeleteCustomerNotificationView(auth_mixins.PermissionRequiredMixin, views.DeleteView):
    model = CustomerNotification
    template_name = 'customer_notification/customer_notification_delete.html'

    permission_required = 'main.change_serviceordernote'

    def get_success_url(self):
        go_to_next = self.request.POST.get('next', '/')
        return go_to_next

class CustomerNotificationDetailView(auth_mixins.PermissionRequiredMixin, views.DetailView):
    model = CustomerNotification
    template_name = 'customer_notification/customer_notification_detail.html'

    permission_required = 'main.view_serviceordernote'


import tensorflow as tf
from tensorflow import keras
import random
import numpy as np
from statistics import mean, median
from collections import Counter
from main import Game as game

def initial_population():
    training_data = []
    scores = []
    accepted_scores = []
    for i in range(initial_games):
        print('Initial Population', ' Simulation: ', i + 1, '/' , initial_games)
        env.reset()
        score = 0
        game_memory = []
        prev_observation = []
        for _ in range(goal_steps):
            action = [0, 0, 0, 0]
            for a in range(0, len(action)):
                action[a] = random.choice([0, 1])
            observation, reward, done, info = env.run(action)
            if len(prev_observation) > 0:
                game_memory.append([prev_observation, action])
            prev_observation = observation
            score += reward
            if done:
                break
        if score >= score_requirement:
            accepted_scores.append(score)
            for data in game_memory:
                training_data.append(data)
        scores.append(score)
    training_data_save = np.array(training_data)
    training_data_name = 'data_gen_0.npy'
    np.save('data/' + training_data_name, training_data_save, allow_pickle=True)
    print('Initial Population', ' Number of accepted scores:', len(accepted_scores))
    print('Initial Population', ' Mean accepted score:', mean(accepted_scores))
    print('Initial Population', ' Median accepted score:', median(accepted_scores))

def recursive_learning(generation):
    training_data = np.load('data/data_gen_' + str(generation - 1) + '.npy', allow_pickle=True)
    model = train_model(training_data)
    model_name = 'model_gen_' + str(generation)
    model.save('models/' + model_name)
    training_data = []
    scores = []
    accepted_scores = []
    for i in range(initial_games):
        print('Generation:', generation, ' Simulation: ', i + 1, '/' , initial_games)
        env.reset()
        score = 0
        game_memory = []
        prev_observation = []
        for _ in range(goal_steps):
            action = [0, 0, 0, 0]
            if len(prev_observation) == 0:
                for a in range(0, len(action)):
                    action[a] = random.choice([0, 1])
            else:
                pred = np.argmax(model.predict(prev_observation))
                action[pred] = 1
            new_observation, reward, done, info = env.run(action)
            prev_observation = np.array([new_observation])
            if len(prev_observation) > 0:
                game_memory.append([new_observation, action])
            score += reward
            if done:
                break
        if score >= score_requirement:
            accepted_scores.append(score)
            for data in game_memory:
                training_data.append(data)
        scores.append(score)
    training_data_save = np.array(training_data)
    training_data_name = 'data_gen_' + str(generation) + '.npy'
    np.save('data/' + training_data_name, training_data_save, allow_pickle=True)
    print('Generation:', generation, ' Number of accepted scores:', len(accepted_scores))
    print('Generation:', generation, ' Mean accepted score:', mean(accepted_scores))
    print('Generation:', generation, ' Median accepted score:', median(accepted_scores))

def neural_network_model(input_size):
    model = keras.Sequential([
    keras.layers.Dense(12, input_dim = input_size, activation = 'sigmoid'),
    keras.layers.Dense(12, activation = 'sigmoid'),
    keras.layers.Dense(4, activation='softmax'),
    keras.layers.Dropout(0.8)
    ])
    opt = keras.optimizers.Adam(learning_rate=LR)
    model.compile(optimizer=opt,
              loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              metrics=['accuracy'])
    model.summary()
    return model

def train_model(training_data, model=False):
    X = np.array([i[0] for i in training_data])

    y = np.array([i[1] for i in training_data])
    if not model:
          model = neural_network_model(input_size = len(X[0]))
    model.fit(X, y, epochs = 5)
    return model

if __name__ == "__main__":
    #training_data = initial_population()
    LR = 1e-3
    env = game()
    goal_steps = 1000
    score_requirement = 0
    initial_games = 100
    generations = 10

    initial_population()
    for g in range (1, generations + 1):
        recursive_learning(g)


import pandas as pd
from scipy import stats

from locan import LocData
from locan.analysis import LocalizationProperty
from locan.analysis.localization_property import _DistributionFits

def test_test_LocalizationProperty_empty(caplog):
    lprop = LocalizationProperty().compute(LocData())
    lprop.fit_distributions()
    lprop.plot()
    lprop.hist()
    assert caplog.record_tuples == [
        ("locan.analysis.localization_property", 30, "Locdata is empty."),
        ("locan.analysis.localization_property", 30, "No results available to fit."),
    ]

def test_DistributionFits(locdata_rapidSTORM_2d):
    lprop = LocalizationProperty().compute(locdata=locdata_rapidSTORM_2d)
    assert repr(lprop) == "LocalizationProperty(loc_property=intensity, index=None)"
    ds = _DistributionFits(lprop)
    ds.fit(distribution=stats.expon)
    # print(ds.parameters)
    # print(ds.__dict__)
    # #print(ds.parameter_dict())
    # ds.plot()
    assert ds.parameters == ["intensity_loc", "intensity_scale"]
    ds.fit(distribution=stats.expon, with_constraints=False, floc=1000)
    assert ds.parameter_dict()["intensity_loc"] == 1000

def test_Localization_property(locdata_rapidSTORM_2d):
    lprop = LocalizationProperty(loc_property="intensity")
    assert not lprop
    lprop.plot()
    lprop.hist()
    lprop.compute(locdata=locdata_rapidSTORM_2d)
    assert lprop
    # print(lprop)
    assert repr(lprop) == "LocalizationProperty(loc_property=intensity, index=None)"
    assert lprop.results.columns == pd.Index(["intensity"], dtype="object")
    assert lprop.distribution_statistics is None
    lprop.plot()
    lprop.hist()
    lprop.fit_distributions()
    assert list(lprop.distribution_statistics.parameter_dict().keys()) == [
        "intensity_loc",
        "intensity_scale",
    ]

    lprop = LocalizationProperty(loc_property="intensity", index="frame").compute(
        locdata=locdata_rapidSTORM_2d
    )
    # print(lprop.results)
    assert lprop.results.index.name == "frame"
    assert lprop.results.columns == pd.Index(["intensity"], dtype="object")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('heritage', '0029_auto_20160219_0349'),
    ]

    operations = [
        migrations.RenameField(
            model_name='interview',
            old_name='interviewer',
            new_name='primary_interviewer',
        ),
        migrations.AlterField(
            model_name='interview',
            name='participant_number',
            field=models.IntegerField(verbose_name='Participant number'),
        ),
        migrations.AlterUniqueTogether(
            name='interview',
            unique_together=set([('phase', 'primary_interviewer', 'type', 'participant_number')]),
        ),
    ]


from fastapi import APIRouter
from app.core.presenter import level as level_pres
from app.core.model import level as level_mod

router = APIRouter()

@router.get('/level')
async def list_level():
    list_levels = {
    "administrator":[{"kode_level":1}],
    "admin_pusat":[{"kode_level":2}],
    "admin_provinsi":[{"kode_level":3}],
    "admin_kabupaten":[{"kode_level":4}],
    "caleg":[{"kode_level":5}],
    "saksi":[{"kode_level":6}],
    "anggota":[{"kode_level":7}],
    }

    return list_levels

@router.post('/level/upgrade')
async def upgrade_level(upgrade: level_mod.UpgradeLevel):
    return await level_pres.handle(upgrade.user_id, upgrade.user_id_target, upgrade.target_level, mode='upgrade')

@router.post('/level/downgrade',)
async def downgrade_level(user_id: int, user_id_target: int, target_level: level_mod.Level, mode = 'downgrade') -> level_mod.Result:
    return await level_pres.handle(user_id, user_id_target, target_level, mode)


import os
import numpy as np
import signal
import logging

from time import sleep
from typing import Union, Optional
from threading import Lock
from rich.progress import Progress
from multiprocessing import Manager, Process

from spt.config import ForwardModelParams, SamplingParams
from spt.modelling.simulation import Simulator, Status
from spt.load_photometry import save_sim, join_partial_results

status_t = tuple[int, Status, int]  # rank, Status, N

class TSStatus:
    def __init__(self, val: Status = Status.STARTING):
        self._lock = Lock()
        self.value: Status = val
        self.n = 0

    def update(self, new: Status, n: Optional[int] = None):
        self._lock.acquire()
        try:
            self.value = new
            if n is not None:
                self.n = n
        finally:
            self._lock.release()

def sim_func(status, idx: int):
    # status is a ListProxy

    # silence all non-error logs:
    log = logging.getLogger()
    log.setLevel(logging.ERROR)

    sp = SamplingParams()
    N = sp.n_samples // sp.concurrency

    sim = Simulator(ForwardModelParams, sp.observation)

    status[0] = Status.LOADED

    theta = np.zeros((N, sim.dim))
    phot = np.zeros((N, sim.phot_dim))

    for i in range(N):
        theta[i], phot[i] = sim.simulate_sample()
        if i % 10 == 0:
            status[0] = Status.SAMPLED
            status[1] = i

    status[0] = Status.SAVING
    status[1] = N

    save_path = os.path.join(sp.save_dir, f'photometry_sim_{N}_{idx}.h5')
    assert isinstance(sim.obs['phot_wave'], np.ndarray)
    save_sim(save_path, theta, sim.model.free_params, phot, sim.obs['phot_wave'])

    # set the status to done before exiting.
    status[0] = Status.DONE
    status[1] = N

def main():

    from mpi4py import MPI
    from spt.utils import splash_screen

    MPIComm = Union[MPI.Intracomm, MPI.Intercomm]

    def root_func(mpi_comm: MPIComm):

        splash_screen()
        sp = SamplingParams()
        C = sp.concurrency
        N = sp.n_samples // C

        if not os.path.exists(sp.save_dir):
            os.makedirs(sp.save_dir)
            logging.info(f'Created results directory {sp.save_dir}')

        sm = Manager()
        status = sm.list([Status.STARTING, 0])

        p = Process(target=sim_func, args=(status, mpi_comm.Get_rank()))
        p.start()

        def interrupt(signum, _):
            logging.error(f'Rank {mpi_comm.Get_rank} received signal {signum}')
            p.terminate()
            mpi_comm.bcast(True)

        map((lambda x: signal.signal(x, interrupt)),
                [signal.SIGTERM, signal.SIGINT, signal.SIGKILL, signal.SIGTRAP])

        logging.info(f'Creating a dataset of size {sp.n_samples} across {C} workers')

        if not os.path.exists(sp.save_dir):
            os.makedirs(sp.save_dir)
            logging.info(f'Created results directory {sp.save_dir}')

        with Progress() as progress:
            tasks = []
            for t in range(sp.concurrency):
                tasks.append(progress.add_task(f'[red]Starting    {t:02}', total=N, start=False))

            done: int = 0
            while done < sp.concurrency:
                sleep(1)

                mpi_comm.bcast(False)

                done = 0
                # all_status = mpi_comm.gather(None)
                all_status = mpi_comm.gather((0, status[0], status[1]))
                assert isinstance(all_status, list)

                for s in all_status:
                    assert len(s) == 3, "Bad status update"
                    tidx, ts, tn = s
                    if ts == Status.STARTING:
                        progress.update(tasks[tidx], completed=tn,
                                        description= f'[red]Starting    {tidx:02}')
                    elif ts == Status.LOADED:
                        progress.reset(tasks[tidx], completed=tn, start=True,
                                       description=f'[dark_orange]Loading SPS {tidx:02}')
                    elif ts == Status.SAMPLED:
                        progress.update(tasks[tidx], completed=tn,
                                        description=f'[green]Running     {tidx:02}')
                    elif ts == Status.SAVING:
                        progress.update(tasks[tidx], completed=tn,
                                        description=f'[green]Saving       {tidx:02}')
                    elif ts == Status.DONE:
                        progress.update(tasks[tidx], completed=tn,
                                        description=f'[white]Done        {tidx:02}')
                        done += 1

        # send TERM signal
        mpi_comm.bcast(True)

        logging.info(f'Joining {sp.concurrency} partial result files')

        join_partial_results(sp.save_dir, sp.n_samples, sp.concurrency)

    def work_func(mpi_comm: MPIComm):

        sm = Manager()
        status = sm.list([Status.STARTING, 0])

        idx = mpi_comm.Get_rank()
        p = Process(target=sim_func, args=(status, idx))
        p.start()

        TERM = False
        while not TERM:
            # block on poll request from root rank
            TERM = mpi_comm.bcast(None)

            # return the status. Note, we may send DONE multiple times.
            mpi_comm.gather((idx, status[0], status[1]))

        if p.is_alive():
            p.terminate()

    mpi_comm: MPIComm = MPI.COMM_WORLD
    mpi_rank = mpi_comm.Get_rank()
    mpi_size = mpi_comm.Get_size()

    if mpi_size == 1:
        logging.warning((
            'Running MPI with only 1 rank.\n'
            'Try increasing SamplingParams.concurrency or check the mpi launch'
            ' command (e.g. mpirun or similar).'))
        raise RuntimeError('Only one MPI rank.')

    if mpi_rank == 0:
        root_func(mpi_comm)
    else:
        work_func(mpi_comm)

if __name__ == '__main__':

    main()


import sys, math, re, time, os, pathlib

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
from copy import deepcopy
from scipy import stats
from scipy import optimize
from scipy import special
from tqdm import tqdm
import pickle

import locale
locale.setlocale(locale.LC_ALL, "");
import warnings
warnings.filterwarnings("ignore");

###################################################################################
BREAKBOUND = pow(10.0, -3);
LOWBOUND   = pow(10.0, -9);
SNUMBER    = pow(10.0, -124);

def expCDF_func(L, x):
    return 1.0 - np.exp(-L*x);
def gausCDF_func(mu, sig, x):
    return 0.5*(1 + math.erf((x - mu)/(sig*np.sqrt(2))));
def expCDF_lambda(L):
    return lambda x : expCDF_func(L, x);
def gausCDF_lambda(mu, sig):
    return lambda x : gausCDF_func(mu, sig, x);

def poissonGetProb(n, L):
    if n == 0: return 1.0;
    return 1.0 - stats.poisson.cdf(n-1, L);
def poissonProb_lambda(n, mu):
    return lambda norm : poissonGetProb(n, norm*mu);
def poissonOpt_lambda(n, mu, alpha):
    return lambda norm : abs(poissonGetProb(n, norm*mu) - alpha);

###################################################################################
def main():
    if len(sys.argv) < 2:
        print("Please input the number of signal data points.");
        print("Run stops.")
        sys.exit(0);
    verbosity = 1;
    sampleN = 3000;
    incBoundary = True;
    binN   = 1000;
    rangeX = [0.0, 5.0];
    rangeG = [1.0, 3.0];

    alpha = 0.9;

    signalN = int(sys.argv[1]);
    #noiseN  = 0;                           #noise number
    noiseN  = 10;             #noise number
    np.random.seed(int(time.time()));
    #np.random.seed(2);

    dataN   = noiseN + signalN;
    signalMu    = 2.0;
    signalSig   = 1.0;
    noiseLambda = 1.0;
    rangeNorm = [0.0, 100.0]; binNNorm = 100;
    if dataN > 60:
        rangeNorm = [0.0, 200.0]; binNNorm = 200;
#reading c0s, cMAXs from pickle
    pickleName = "pickleRef/c0cMAXUpperBounds.pickle";
    df = pd.read_pickle(pickleName);
    df = df[df["signalN"] == signalN];
    df = df[df["noiseN"] == noiseN];
#    df = df[df["sampleN"] == sampleN];
    c0s   = df["c0s"].to_numpy()[0];
    cMAXs = df["cMAXs"].to_numpy()[0];
#Poisson bound
    if verbosity >= 1:
        print("Sampling the lower bound with signalN=" + str(signalN) + ":");
    nbins = np.linspace(rangeX[0], rangeX[1], binN+1)[:-1];
    signalCDF = gausCDF_lambda(signalMu, signalSig);    #signal shape
    mu = signalCDF(rangeG[1]) - signalCDF(rangeG[0]);
    poissons = [];
    for s in tqdm(range(sampleN)):
        noisePDF  = np.random.exponential(1.0/noiseLambda, noiseN);
        signalPDF = np.random.normal(signalMu, signalSig, signalN);
        dataPDF   = np.concatenate((noisePDF, signalPDF), axis=0);
        #getting the number of events
        dataPDF = [x for x in dataPDF if ((rangeG[0]<=x) and (x<=rangeG[1]))];
        eventN = len(dataPDF);

        optFactor = 5.0;
        optRange  = [eventN/optFactor, optFactor*eventN];
        #poisson
        poissonOpt = poissonOpt_lambda(eventN, mu, alpha);
        normOpt = optimize.minimize_scalar(poissonOpt,
                    method="bounded", bounds=(optRange[0], optRange[1]),\
                    options={"xatol": LOWBOUND});
        poissonProb = poissonProb_lambda(eventN, mu);
        normUpBd = normOpt.x;
        poissons.append(normUpBd);
        if verbosity >= 2:
            print("S =", s);
            print("poi: opt norm =", normUpBd);
            print("poi: (n, mu)  =", [eventN, mu]);
            print("poi: conf (set, opt) =", [alpha, poissonProb(normUpBd)]);
    if verbosity >= 1:
        print("");
        print("c0:", c0s, "\n");
        print("cMAX:", cMAXs, "\n");
        print("Poisson:", poissons, "\n");
#pickle save
    pathlib.Path("pickle").mkdir(parents=True, exist_ok=True);
    pickleName = "pickle/c0cMAXUpperBoundsPoisson.pickle";
    try:
        df = pd.read_pickle(pickleName);
    except (OSError, IOError) as e:
        columnNames=["signalN", "noiseN", "sampleN", "incBoundary",\
                     "c0s", "cMAXs", "poissons"];
        df = pd.DataFrame(columns = columnNames);
    bdData = {"signalN":     signalN, \
              "noiseN":      noiseN, \
              "sampleN":     sampleN, \
              "incBoundary": incBoundary,\
              "c0s":         c0s,\
              "cMAXs":       cMAXs,\
              "poissons":    poissons};
    cond = (df["signalN"]==signalN) & (df["noiseN"]==noiseN);
    if df[cond].empty == False:
        df = df.drop(df[cond].index);
    df = df.append(bdData, ignore_index=True);
    df.to_pickle(pickleName);
    if verbosity >= 1:
        print("The following files have been updated:");
        print("    ", pickleName);
#hist
    noiseHist = np.zeros(binN);
    signalHist = np.zeros(binN);
    dataHist = np.zeros(binN);
    for x in noisePDF:
        if rangeX[0] < x and x < rangeX[1]:
            histIdx = int(np.floor(binN*(x-rangeX[0])/(rangeX[1]-rangeX[0])));
            noiseHist[histIdx] += 1;
    for x in signalPDF:
        if rangeX[0] < x and x < rangeX[1]:
            histIdx = int(np.floor(binN*(x-rangeX[0])/(rangeX[1]-rangeX[0])));
            signalHist[histIdx] += 1;
    for x in dataPDF:
        if rangeX[0] < x and x < rangeX[1]:
            histIdx = int(np.floor(binN*(x-rangeX[0])/(rangeX[1]-rangeX[0])));
            dataHist[histIdx] += 1;
    nbinsNorm = np.linspace(rangeNorm[0], rangeNorm[1], binNNorm+1)[:-1];
    c0Hist = np.zeros(binNNorm);
    cMAXHist = np.zeros(binNNorm);
    for norm in c0s:
        if rangeNorm[0] < norm and norm < rangeNorm[1]:
            histIdx = int(np.floor(binNNorm*(norm-rangeNorm[0])/\
                                            (rangeNorm[1]-rangeNorm[0])));
            c0Hist[histIdx] += 1;
    for norm in cMAXs:
        if rangeNorm[0] < norm and norm < rangeNorm[1]:
            histIdx = int(np.floor(binNNorm*(norm-rangeNorm[0])/\
                                            (rangeNorm[1]-rangeNorm[0])));
            cMAXHist[histIdx] += 1;
    #poisson upper bounds be discrete because eventN is discrete
    factor = 2;
    binNNorm = int(binNNorm/factor);
    nbinsNormPoi = np.linspace(rangeNorm[0], rangeNorm[1], binNNorm+1)[:-1];
    poissonHist = np.zeros(binNNorm);
    for norm in poissons:
        if rangeNorm[0] < norm and norm < rangeNorm[1]:
            histIdx = int(np.floor(binNNorm*(norm-rangeNorm[0])/\
                                            (rangeNorm[1]-rangeNorm[0])));
            poissonHist[histIdx] += 1;
    for i, entry in enumerate(poissonHist):
        poissonHist[i] = entry/factor;
#plots
    fig = plt.figure(figsize=(18, 7));
    gs = gridspec.GridSpec(1, 2);
    ax0 = fig.add_subplot(gs[0]);
    ax1 = fig.add_subplot(gs[1]);
    #plot 0
    ax0.plot(nbins,dataHist,  linewidth=2,color="black",drawstyle="steps-post");
    ax0.plot(nbins,noiseHist, linewidth=2,color="red",  drawstyle="steps-post");
    ax0.plot(nbins,signalHist,linewidth=2,color="blue", drawstyle="steps-post");
    ax0.set_title("Generated Noise(red)&Signal(blue)", fontsize=24, y=1.03);
    ax0.set_xlabel("x", fontsize=18);
    ax0.set_ylabel("count", fontsize=18);
    ax0.set_xlim(rangeX[0], rangeX[1]);
    ax0.axhline(y=0, xmin=0, xmax=1, color="black", linewidth=2);
    #plot 1
    ax1.plot(nbinsNormPoi, poissonHist, color="purple", alpha=1,\
             linewidth=2, drawstyle="steps-post");
    ax1.plot(nbinsNorm, c0Hist, color="green", alpha=1,\
             linewidth=2, drawstyle="steps-post");
    ax1.plot(nbinsNorm, cMAXHist, color="orange", alpha=1,\
             linewidth=2, drawstyle="steps-post");
    ax1.set_title("Signal Upper Bound, C0(green)&CMAX(orange)&Poi(purple)", \
                  fontsize=16, y=1.03);
    ax1.set_xlabel("signal upper bound", fontsize=18);
    ax1.set_ylabel("count", fontsize=18);
    ax1.set_xlim(rangeNorm[0], rangeNorm[1]);
    ax1.axhline(y=0, xmin=0, xmax=1, color="black", linewidth=2);
    ax1.axvline(x=signalN, ymin=0, ymax=1, color="blue", linewidth=2, alpha=0.5);
#save plots
    pathlib.Path("figure/maxGapExp").mkdir(parents=True, exist_ok=True);
    filenameFig = "figure/maxGapExp/maxGapS";
    filenameFig = filenameFig + str(signalN) + "N" + str(noiseN) + "Poisson.png";
    gs.tight_layout(fig);
    plt.savefig(filenameFig);

    if verbosity >= 1:
        print("Creating the following files:");
        print("    ", filenameFig);

###################################################################################
if __name__ == "__main__":
    print("\n##############################################################Head");
    main();
    print("##############################################################Tail");


# -*- coding: utf-8 -*-
"""
Created on Fri Nov 30 23:18:32 2018

@author: Christine, Patrick
"""

import matplotlib.pyplot as plt
import numpy as np
import scipy
import praktikum.cassy as cassy
import praktikum.analyse as analyse

def gauss(x, m, s):
    return 1/np.sqrt(np.pi*2)/s*np.exp(-(x-m)**2/(2*s**2))

def round_good(m,s,err): #für obere Abschätzung der Fehler
    '''
    Passt Mittelwert- und Standardabweichungsnachkommastellen an den Fehler an (2 signifikante Stellen des Fehlers)
    m=mean
    s=std
    err=std/sqrt(len)
    '''
    i=2
    test=list('{:.20f}'.format(err))
    while test[i]=='0':
        i+=1
    return np.round([m,s,err],i)

plt.close('all')

data = cassy.CassyDaten("gekoppelt/Rauschmessung.lab")
t = data.messung(1).datenreihe("t").werte
M = data.messung(1).datenreihe("U_A1").werte #Pendelkörper (Masse)
St = data.messung(1).datenreihe("U_B1").werte #Stange allein

#----------------------------#
#Auswertung der Rauschmessung#
#----------------------------#

#Mittelwerte, Standardabweichungen und Fehler
mM,sM,errM=round_good(np.average(M),np.std(M),np.std(M)/np.sqrt(len(M)))
mSt,sSt,errSt=round_good(np.average(St),np.std(St),np.std(St)/np.sqrt(len(St)))

if __name__ == "__main__":

#------------------------------------------------------------
#Erstes Pendel
    print("Spannungsverteilung 1")
    print('m='+str(mM)+u' V, std='+str(sM)+u' V, err='+str(errM)+u' V')

    #Histogramm
    weightsM=np.ones_like(M)/float(len(M))
    plt.hist(M,7, normed=0, weights=weightsM)
    x=np.arange(np.average(M)-0.03, np.average(M)+0.03, 0.001)
    #plt.plot(x, gauss(x, np.average(M), np.std(M)))
    plt.title(u"Histogramm der Ruhelage der Pendelstange 1 \n $\sigma$=" + str(sM)+"$V$, $\mu$="+str(mM)+ "$V$, $\sigma_{\mu} = $"+str(errM)+"$V$")
    plt.xlabel(u"U/V")
    plt.ylabel('Relatives Vorkommen')
    plt.savefig("Images/Rauschmessung_stange_1_histo.jpg")
    plt.axvline(np.mean(M),color='red',linestyle='--')
    plt.figure()

    #Datenplot
    plt.plot(t, M)
    plt.title(u"Rauschmessung Ruhelage Pendel 1")
    plt.ylabel(u"U/V")
    plt.xlabel('Zeit/s')
    plt.plot(t,np.array([np.mean(M)]*len(t)),color='darkorange')
    plt.savefig("Images/Rauschmessung_stange_1.jpg")
    plt.figure()

#------------------------------------------------------------
#Zweites Pendel
    print("==="*20)
    print("Spannungsverteilung 2")
    print('m='+str(mSt)+' V, std='+str(sSt)+' V, err='+str(errSt)+' V')

    #Histogramm
    weightsSt=np.ones_like(M)/float(len(M))
    plt.hist(St, 7, normed=0,weights=weightsSt)
    x=np.arange(min(St), max(St), 0.0001)
    #plt.plot(x, gauss(x, np.average(St), np.std(St)))
    plt.title(u"Histogramm der Ruhelage der Pendelstange 2 \n $\sigma$=" + str(sSt)+" V, $\mu$="+str(mSt)+" V, $\sigma_{\mu} = $" +str(errSt)+" V")
    plt.xlabel(u"U/V")
    plt.ylabel('Relatives Vorkommen')
    plt.savefig("Images/Rauschmessung_stange_2_hist.jpg")
    plt.axvline(np.mean(St),color='red',linestyle='--')
    plt.figure()

    #Datenplot
    plt.plot(t, St)
    plt.ylabel(u"U/V")
    plt.xlabel('Zeit/s')
    plt.title("Rauschmessung Ruhelage Pendel 2")
    plt.plot(t,np.array([np.mean(St)]*len(t)),color='darkorange')
    plt.savefig("Images/Rauschmessung_stange_2.jpg")


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import torch.nn.functional as F
import cv2

from pysot.core.config import cfg
from pysot.utils.anchor import Anchors
from pysot.tracker.base_tracker import SiameseTracker
from matplotlib import pyplot as plt
from .external.nms import soft_nms

class SiamCORNERTracker(SiameseTracker):
    def __init__(self, model):
        super(SiamCORNERTracker, self).__init__()
        self.score_size = (cfg.TRACK.INSTANCE_SIZE - cfg.TRACK.EXEMPLAR_SIZE) // \
            cfg.ANCHOR.STRIDE + 1 + cfg.TRACK.BASE_SIZE
        self.anchor_num = len(cfg.ANCHOR.RATIOS) * len(cfg.ANCHOR.SCALES)
        #hanning = np.hanning(self.score_size)
        window = np.hanning(cfg.corners.hanning_score_size)
        #window = np.outer(hanning, hanning)
        self.window = np.tile(window.flatten(), cfg.corners.hanning_num)
        #self.anchors = self.generate_anchor(self.score_size)
        self.model = model
        self.model.eval()

    def generate_anchor(self, score_size):
        anchors = Anchors(cfg.ANCHOR.STRIDE,
                          cfg.ANCHOR.RATIOS,
                          cfg.ANCHOR.SCALES)
        anchor = anchors.anchors
        x1, y1, x2, y2 = anchor[:, 0], anchor[:, 1], anchor[:, 2], anchor[:, 3]
        anchor = np.stack([(x1+x2)*0.5, (y1+y2)*0.5, x2-x1, y2-y1], 1)
        total_stride = anchors.stride
        anchor_num = anchor.shape[0]
        anchor = np.tile(anchor, score_size * score_size).reshape((-1, 4))
        ori = - (score_size // 2) * total_stride
        xx, yy = np.meshgrid([ori + total_stride * dx for dx in range(score_size)],
                             [ori + total_stride * dy for dy in range(score_size)])
        xx, yy = np.tile(xx.flatten(), (anchor_num, 1)).flatten(), \
            np.tile(yy.flatten(), (anchor_num, 1)).flatten()
        anchor[:, 0], anchor[:, 1] = xx.astype(np.float32), yy.astype(np.float32)
        return anchor

    def _convert_bbox(self, delta, anchor):
        delta = delta.permute(1, 2, 3, 0).contiguous().view(4, -1) # delta:[4,3125]
        delta = delta.data.cpu().numpy()

        delta[0, :] = delta[0, :] * anchor[:, 2] + anchor[:, 0] # x
        delta[1, :] = delta[1, :] * anchor[:, 3] + anchor[:, 1] # y
        delta[2, :] = np.exp(delta[2, :]) * anchor[:, 2] # w
        delta[3, :] = np.exp(delta[3, :]) * anchor[:, 3] # h
        return delta

    def _convert_score_att(self, atts, attentions):
        att_score = []
        att_loc = []
        for att, attention in zip(atts, attentions):
            attention_ = attention.repeat(1, 5, 1, 1) # 复制5份[1,1,25,25]-->[1,5,25,25] 5份是因为5种不同的anchors
            attention_ = attention_.permute(1, 2, 3, 0).contiguous().view(1, -1).permute(1, 0) # [3125,1]
            att_score.append(F.softmax(attention_, dim=0).data[:, 0].cpu().numpy()) # [3125,] 注意力特征经过softmax激活函数
            A = []
            for i in range(len(att)): # 注意标注的位置在复制的5份对应的位置
                a_0 = (att[i][0]*25 + att[i][1]).astype(int) # 转化为[3125,]后的位置
                a_1 = a_0 + 25*25*1
                a_2 = a_1 + 25*25*1
                a_3 = a_2 + 25*25*1
                a_4 = a_3 + 25*25*1
                A.append(a_0)
                A.append(a_1)
                A.append(a_2)
                A.append(a_3)
                A.append(a_4)
            att_loc.append(A)

        return att_loc, att_score

    def _convert_score(self, score, att_locs, att_scores):
        W = cfg.atts.W # 注意力机制权值
        score = score.permute(1, 2, 3, 0).contiguous().view(2, -1).permute(1, 0) # [1,10,25,25]-->[3125,2] 后5个为正样本得分
        score = F.softmax(score, dim=1).data[:, 1].cpu().numpy() # [3125,] dim=1 对每一行进行softmax
        for att_loc, att_score, w in zip(att_locs, att_scores, W):
            for i in range(len(att_loc)):
                score[att_loc[i]] += w * att_score[att_loc[i]]
        return score

    def _bbox_clip(self, cx, cy, width, height, boundary):
        cx = max(0, min(cx, boundary[1]))
        cy = max(0, min(cy, boundary[0]))
        width = max(10, min(width, boundary[1]))
        height = max(10, min(height, boundary[0]))
        return cx, cy, width, height

    def rescale_dets_(self, dets_):
        input_size = cfg.corners.input_size  # [255,255]
        output_size = cfg.corners.output_size  # [25,25]

        ratios = [o / i for o, i in zip(output_size, input_size)]
        detect = []
        for dets in dets_:
            dets[..., 0:4:2] /= ratios[1]  # bbox在[255,255,3]中的坐标
            dets[..., 1:4:2] /= ratios[0]
            detect.append(dets)
        return detect

    def Centertocorner(self, det):
        det[:, :, 0] = det[:, :, 0] - det[:, :, 2] // 2 # x1
        det[:, :, 1] = det[:, :, 1] - det[:, :, 3] // 2  # y1
        det[:, :, 2] = det[:, :, 0] + det[:, :, 2]  # x2
        det[:, :, 3] = det[:, :, 1] + det[:, :, 3]  # y2
        return det

    def init(self, img, bbox):
        """
        args:
            img(np.ndarray): BGR image
            bbox: (x, y, w, h) bbox
        """
        self.center_pos = np.array([bbox[0]+(bbox[2]-1)/2, # center_x, center_y
                                    bbox[1]+(bbox[3]-1)/2])
        self.size = np.array([bbox[2], bbox[3]]) # w,h

        # calculate z crop size
        w_z = self.size[0] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(self.size)
        h_z = self.size[1] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(self.size)
        s_z = round(np.sqrt(w_z * h_z))

        # calculate channle average
        self.channel_average = np.mean(img, axis=(0, 1))

        # get crop
        z_crop = self.get_subwindow(img, self.center_pos, # self.center_pos:ground_truth中心点位置
                                    cfg.TRACK.EXEMPLAR_SIZE, # 按照ground_truth中心点裁剪为127*127的图片
                                    s_z, self.channel_average) # image:[1024,1024,3]-->z_crop:[1,3,127,127]
        self.model.template(z_crop)

    def track(self, img):
        """
        args:
            img(np.ndarray): BGR image
        return:
            bbox(list):[x, y, width, height]
        """
        w_z = self.size[0] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(self.size) # self.size[0]:bbox宽  w+p
        h_z = self.size[1] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(self.size) # self.size[1]:bbox高  h+p
        s_z = np.sqrt(w_z * h_z) # 对应论文中公式14
        scale_z = cfg.TRACK.EXEMPLAR_SIZE / s_z
        s_x = s_z * (cfg.TRACK.INSTANCE_SIZE / cfg.TRACK.EXEMPLAR_SIZE)
        x_crop = self.get_subwindow(img, self.center_pos,  # x_crop:[1,3,255,255] 按照中心点进行裁剪
                                    cfg.TRACK.INSTANCE_SIZE,
                                    round(s_x), self.channel_average)
        '''
        # 画图
        img1 = x_crop.squeeze(0)
        img_1 = img1.transpose(1, 0).transpose(2, 1).cpu().numpy()
        a = cv2.imwrite("4.jpg", img_1)  # save picture
        plt.axis('off')
        plt.imshow(img_1[:, :, 1])
        plt.show()
        '''

        outputs = self.model.track(x_crop) # cls:[1,10,25,25] loc:[1,20,25,25]
        dets_ = outputs['bbox']
        dets_1 = self.rescale_dets_(dets_)  # [25,25]-->[255,255] 对应的bbox缩放比例 [x1,y1,x2,y2]

        det = []
        for dets_2 in dets_1:
            dets_2 = dets_2.cpu()
            dets_2 = dets_2.detach().numpy()
            dets_2 = np.squeeze(dets_2)
            det.append(dets_2)

        detectect = np.concatenate((det[0], det[1]))
        det = np.concatenate((detectect, det[2])) # 在[255,255]中的位置，[x1,y1,x2,y2]

        def change(r): # r:[3125,]
            return np.maximum(r, 1. / r)

        def sz(w, h):
            pad = (w + h) * 0.5
            return np.sqrt((w + pad) * (h + pad)) # 对应论文中公式(15)

        # scale penalty
        s_c = change(sz(det[:, 2], det[:, 3]) / # s_c:[3125,]
                     (sz(self.size[0]*scale_z, self.size[1]*scale_z)))# 上一帧bbox在填充之后的

        # aspect ratio penalty
        r_c = change((self.size[0]/self.size[1]) / # self.size:bbox代表上一帧宽，高比
                     (det[:, 2]/det[:, 3]))

        penalty = np.exp(-(r_c * s_c - 1) * cfg.TRACK.PENALTY_K)  # penalty:[3125,]
        score = det[:, 5]
        #pscore = penalty * score  # 对相邻两帧之间过大的形状和尺度变化进行抑制

        # window penalty
        #pscore = pscore * (1 - cfg.TRACK.WINDOW_INFLUENCE) + \
        #         self.window * cfg.TRACK.WINDOW_INFLUENCE  # a cosine window is added to suppress the large displacement
        best_idx = np.argmax(score)
        bbox = det[best_idx, :4] / scale_z  # pred_bbox:[4,3125] 对应padding [x1,y1,x2,y2]
        #lr = penalty[best_idx] * score[best_idx] * cfg.TRACK.LR
        lr = 0.13104

        bbox[2], bbox[3] = bbox[2] - bbox[0], bbox[3] - bbox[1]  # 对应于padding宽，高
        bbox[0], bbox[1] = bbox[0] + bbox[2] // 2, bbox[1] + bbox[3] // 2  # 中心点坐标x,中心点坐标y
        s_x = round((s_x + 1) // 2)
        bbox[0], bbox[1] = bbox[0] - s_x, bbox[1] - s_x  # 中心点坐标x,y的偏移量
        bbox[0] = bbox[0] + self.center_pos[0]  # 对应于原图中心点在原图坐标x
        bbox[1] = bbox[1] + self.center_pos[1]  # 中心点在原图坐标y

        # smooth bbox
        width = self.size[0] * (1 - lr) + bbox[2] * lr  # self.size[0]:bbox代表上一帧宽
        height = self.size[1] * (1 - lr) + bbox[3] * lr

        # clip boundary
        cx, cy, width, height = self._bbox_clip(bbox[0], bbox[1], width,  # 目标框框不能超过图片边界
                                                height, img.shape[:2])

        # udpate state
        self.center_pos = np.array([cx, cy])
        self.size = np.array([width, height])

        bbox = [cx - width / 2,
                cy - height / 2,
                width,
                height]
        best_score = score[best_idx]
        return {
            'bbox': bbox,
            'best_score': best_score
        }

'''

        detection = []  # 对应于padding后的坐标
        for det1 in dets_1:
            det1[..., 0:4] /= scale_z
            detection.append(det1) # 对应于padding后的坐标 x1,y1,x2,y2

        det = []

        for det_ in detection: # 在原图位置
            det_[:, :, 2], det_[:, :, 3] = det_[:, :, 2]-det_[:, :, 0], det_[:, :, 3]-det_[:, :, 1] # 宽，高
            det_[:, :, 0], det_[:, :, 1] = det_[:, :, 0] + det_[:, :, 2] // 2, det_[:, :, 1] + det_[:, :, 3] // 2# 中心点坐标x,中心点坐标y
            s_x = round((s_x+1)//2)
            det_[:, :, 0], det_[:, :, 1] = det_[:, :, 0] - s_x, det_[:, :, 1] - s_x  # 中心点坐标x,y的偏移量
            det_[:, :, 0] = det_[:, :, 0] + self.center_pos[0] # 中心点在原图坐标x
            det_[:, :, 1] = det_[:, :, 1] + self.center_pos[1] # 中心点在原图坐标y
            detection = self.Centertocorner(det_) # x1,y1,x2,y2

            det.append(detection)

        detectection = []
        for detect in det: # Soft nms
            detect = detect.cpu()
            detect = detect.detach().numpy()
            detect = np.squeeze(detect)
            keep_inds = soft_nms(detect, Nt=cfg.corners.nms_threshold, method=cfg.corners.nms_algorithm,
                                 sigma=0.7)  # remove redundant ones by applying Soft-NMS
            det_ = detect[keep_inds, 0:5]
            detectection.append(det_)

        detectect = np.concatenate((detectection[0], detectection[1]))
        det = np.concatenate((detectect, detectection[2]))

        cx = bbox[0] + self.center_pos[0]
        cy = bbox[1] + self.center_pos[1]

        # clip boundary
        cx, cy, width, height = self._bbox_clip(cx, cy, width, # 目标框框不能超过图片边界
                                                height, img.shape[:2])

        # udpate state
        self.center_pos = np.array([cx, cy])
        self.size = np.array([width, height])

        bbox = [cx - width / 2,
                cy - height / 2,
                width,
                height]
        best_score = score[best_idx]
        return {
                'bbox': bbox,
                'best_score': best_score
               }
'''


from django.shortcuts import redirect
from django.template.response import TemplateResponse

from ...account.forms import get_address_form
from ...account.models import Address
from ..forms import AnonymousUserShippingForm, ShippingAddressesForm

def anonymous_user_shipping_address_view(request, checkout):
    """Display the shipping step for a user who is not logged in."""
    address_form, preview = get_address_form(
        request.POST or None, country_code=request.country.code,
        autocomplete_type='shipping',
        initial={'country': request.country.code},
        instance=checkout.shipping_address)

    user_form = AnonymousUserShippingForm(
        not preview and request.POST or None, initial={'email': checkout.email}
        if not preview else request.POST.dict())
    if all([user_form.is_valid(), address_form.is_valid()]):
        checkout.shipping_address = address_form.instance
        checkout.email = user_form.cleaned_data['email']
        return redirect('checkout:shipping-method')
    return TemplateResponse(
        request, 'checkout/shipping_address.html', context={
            'address_form': address_form, 'user_form': user_form,
            'checkout': checkout})

def user_shipping_address_view(request, checkout):
    """Display the shipping step for a logged in user.

    In addition to entering a new address the user has an option of selecting
    one of the existing entries from their address book.
    """
    data = request.POST or None
    additional_addresses = request.user.addresses.all()
    checkout.email = request.user.email
    shipping_address = checkout.shipping_address

    if shipping_address is not None and shipping_address.id:
        address_form, preview = get_address_form(
            data, country_code=request.country.code,
            initial={'country': request.country})
        addresses_form = ShippingAddressesForm(
            data, additional_addresses=additional_addresses,
            initial={'address': shipping_address.id})
    elif shipping_address:
        address_form, preview = get_address_form(
            data, country_code=shipping_address.country.code,
            instance=shipping_address)
        addresses_form = ShippingAddressesForm(
            data, additional_addresses=additional_addresses)
    else:
        address_form, preview = get_address_form(
            data, initial={'country': request.country},
            country_code=request.country.code)
        addresses_form = ShippingAddressesForm(
            data, additional_addresses=additional_addresses)

    if addresses_form.is_valid() and not preview:
        if (addresses_form.cleaned_data['address'] !=
                ShippingAddressesForm.NEW_ADDRESS):
            address_id = addresses_form.cleaned_data['address']
            checkout.shipping_address = Address.objects.get(id=address_id)
            return redirect('checkout:shipping-method')
        elif address_form.is_valid():
            checkout.shipping_address = address_form.instance
            return redirect('checkout:shipping-method')
    return TemplateResponse(
        request, 'checkout/shipping_address.html', context={
            'address_form': address_form, 'user_form': addresses_form,
            'checkout': checkout,
            'additional_addresses': additional_addresses})


from django.shortcuts import render, redirect, get_object_or_404
from django.utils import timezone
from django.contrib.auth.decorators import login_required

from .models import Post, Image
from .forms import PostForm, ImageForm

from django.core.exceptions import ObjectDoesNotExist

def index(request):
	posts = Post.objects.filter(published_date__lte=timezone.now()).order_by('-published_date')
	list = []
	for post in posts:
		image = Image.objects.filter(post_id=post.pk)
		url = ''
		if image.count() != 0:
			url = image[0].image.url
		item = {'post': post, 'image': url}
		list.append(item)
	return render(request, 'index.html', {'list': list})

@login_required
def upload(request):
	if request.method == 'POST':
		form = PostForm(request.POST)
		form_image = ImageForm(request.POST, request.FILES)
		if form.is_valid():
			post = form.save(commit=False)
			post.author = request.user
			post.published_date = timezone.now()
			post.save()
			url = ''
			if form_image.is_valid():
				image = form_image.save(commit=False)
				image.post_id = post
				url = image.image.url
				image.save()
			# post.thumbnail = url
			# post.save()
			return redirect('myapp.views.index')
	return render(request, 'upload.html')

def detail(request, post_id):
	post = get_object_or_404(Post, pk=post_id)
	images = Image.objects.filter(post_id=post_id)
	return render(request, 'detail.html', {'post': post, 'images': images})

def delete(request, post_id):
	post = get_object_or_404(Post, pk=post_id)
	post.delete()
	return redirect('myapp.views.index')


import typing as _t
from decimal import Decimal
from dataclasses import dataclass
from django.db import models
from django.contrib.contenttypes.models import ContentType
from . import stripe_api

# Abstract Models

class ContentTypeModel(models.Model):
    """An abstract base class for models that need to reference other models
    by their content type.
    """
    content_type = models.ForeignKey(
        ContentType,
        on_delete=models.PROTECT,
        related_name='+'
    )
    object_id = models.CharField(max_length=255, blank=True, null=True)

    class Meta:
        abstract = True

    @property
    def object(self) -> models.Model:
        if not hasattr(self, '_object'):
            obj = None
            try:
                obj = self.content_type.get_object_for_this_type(
                    pk=self.object_id
                )
            except models.ObjectDoesNotExist:
                pass
            self._object = obj
        return self._object

    @classmethod
    def get_for_instance(
        cls,
        instance: models.Model
    ) -> _t.Union['PaymentHistory', None]:
        """Retrieves the payment history record for a given instance if it
        exists.

        Args:
            instance (models.Model): The instance for which to retrieve the
                payment history record.
        """
        try:
            return cls.objects.get(
                content_type=ContentType.objects.get_for_model(instance),
                object_id=instance.pk
            )
        except cls.DoesNotExist:
            return None

# Models
class PaymentHistory(ContentTypeModel):
    pay_intent_id = models.CharField(max_length=255, unique=True)
    amount = models.DecimalField(max_digits=10, decimal_places=2)
    card_type = models.CharField(max_length=20)
    last_4_digits = models.CharField(max_length=4)
    exp_month = models.PositiveIntegerField(verbose_name='Expiry month')
    created_on = models.DateTimeField(auto_now_add=True)
    success = models.BooleanField()

    class Meta:
        ordering = ['-created_on']
        verbose_name = 'Payment History'
        verbose_name_plural = 'Payment History'

    @classmethod
    def create_for_instance(
        cls,
        instance: models.Model,
        amount: Decimal
    ) -> 'PaymentHistory':
        """Creates a payment history record for a given instance.

        Args:
            instance (models.Model): The instance for which to create the
                payment history record.
            amount (Decimal): The amount of the payment.
        """
        rec = cls.objects.create(
            content_type=ContentType.objects.get_for_model(instance),
            object_id=instance.pk,
            amount=amount
        )
        rec.save()
        return rec

class Product(ContentTypeModel):
    product_id = models.CharField(
        max_length=255,
        unique=True,
        verbose_name='Stripe Product ID'
    )

class Price(models.Model):

    price_id = models.CharField(
        max_length=255,
        unique=True,
        verbose_name='Stripe Price ID',
    )
    product = models.ForeignKey(
        Product,
        on_delete=models.PROTECT,
        related_name='prices'
    )
    amount = models.DecimalField(max_digits=10, decimal_places=2)
    recurring = models.BooleanField()

    class Meta:
        verbose_name = 'Price'
        verbose_name_plural = 'Prices'

# Model Mixins

class PaymentRecordMixin:
    @property
    def payment_history(self) -> _t.Union['PaymentHistory', None]:
        """Retrieves the payment history record for a given instance if it
        exists.
        """
        return PaymentHistory.get_for_instance(self)

# Data Classes

@dataclass
class ModelInfo:
    app_name: str
    model_name: str
    object_id: int
    content_type_id: int
    amount_attr: str

    def get_object(self) -> models.Model:
        return ContentType.objects.get_for_id(
            self.content_type_id
        ).get_object_for_this_type(
            pk=self.object_id
        )

    def get_amount(self, as_integral: bool = False) -> _t.Union[Decimal, int]:
        """Returns the amount of the instance.
        Args:
            as_integral (bool): Whether to return the amount as an integer. If
                True, the amount will be multiplied by 100.
        """

        amount = getattr(self.get_object(), self.amount_attr)
        if as_integral:
            return int(amount * 100)
        return amount

    @property
    def product(self) -> Product:
        """Retrieves the product for a given instance, otherwise creates a
        new product.
        """

    @property
    def metadata(self) -> _t.Dict[str, str]:
        """Returns the metadata for a given instance to be used in the Stripe
        API.
        """
        return {
            'content_type_id': self.content_type_id,
            'object_id': self.object_id,
        }

    @classmethod
    def from_instance(
        cls,
        model: models.Model,
        amount_attr: str,
    ) -> 'ModelInfo':
        """Creates a ModelInfo object from a model instance.

        Args:
            model (models.Model): The model instance.
            amount_attr (str): The name of the attribute that contains the
                amount.
        """
        return ModelInfo(
            app_name=model._meta.app_label,
            model_name=model._meta.model_name,
            object_id=model.pk,
            content_type_id=ContentType.objects.get_for_model(model).pk,
            amount_attr=amount_attr,
        )


import matplotlib.pyplot as plt
import networkx as nx

class Tree:
    def __init__(self, key, left=None, right=None):
        self.key = key
        self.left = left
        self.right = right

def inorder_layout(tree):
    pos = {}
    def traverse(node, depth=0, pos=pos):
        if node.left:
            traverse(node.left, depth + 1)
        pos[node.key] = (len(pos), -depth)  # horizontal position is number of nodes visited so far
        if node.right:
            traverse(node.right, depth + 1)
    traverse(tree)
    return pos

def build_graph(tree):
    edges = []
    labels = {}
    def traverse(node):
        nonlocal edges, labels
        if node is None:
            return
        if node.left:
            edges.append((node.key, node.left.key))
            labels[(node.key, node.left.key)] = 'yes'
            traverse(node.left)
        if node.right:
            edges.append((node.key, node.right.key))
            labels[(node.key, node.right.key)] = 'not'
            traverse(node.right)
    traverse(tree)
    return edges, labels

# construct a more complicated tree for demonstration
t = Tree('A not match',
         Tree('B not match', Tree('D not match', Tree('H match'), Tree('I match')), Tree('E match', Tree('J match'))),
         Tree('C match', Tree('F match', Tree('K match')), Tree('G match', Tree('L match'),Tree('MD match', Tree('MH match'), Tree('MI match')))))

edges, edge_labels = build_graph(t)
pos = inorder_layout(t)

G = nx.DiGraph()
G.add_edges_from(edges)
plt.figure(figsize=(10, 10))
nx.draw(G, pos, with_labels=True, arrows=False, node_size=3000, node_color='white', font_weight='bold', linewidths=2.0)
nx.draw_networkx_edge_labels(G, pos, edge_labels=edge_labels, font_size=10)
plt.show()


from enum import Enum

Month = Enum('Month', ('Jan', 'Feb', 'Mar', 'Apr',
                       'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'))

for name, member in Month.__members__.items():
    print(name, '-->', member, ',', member.value)

# value属性则是自动赋给成员的int常量，默认从1开始计数。

# 如果需要更精确地控制枚举类型，可以从Enum派生出自定义类：
from enum import Enum, unique

# @unique装饰器可以帮助我们检查保证没有重复值。
@unique
class Weekday(Enum):
    Sun = 0  # Sun的value被设定为0
    Mon = 1
    Tue = 2
    Wed = 3
    Thu = 4
    Fri = 5
    Sat = 6

# 访问这些枚举类型可以有若干种方法：
day1 = Weekday.Mon
print(day1)  # Weekday.Mon
print(Weekday.Tue)  # Weekday.Tue
print(Weekday['Tue'])  # Weekday.Tue
print(Weekday.Mon.value)  # 1
print(day1 == Weekday.Mon)  # True

"""  练习不会做  """
#
# class Gender(Enum):
#     Male = 0
#     Female = 1
#
# class Student(object):
#     def __init__(self, name, gender):
#         self.name = name
#         self.gender = None
#         if gender == Gender.Male:
#             print(self.gender == Gender.Male.value)
#         if gender == Gender.Female:
#             print(self.gender == Gender.Female.value)
#         # else:
#         #     raise ValueError("gender must be Male or Female ！")
#
#     def __str__(self):
#         print(self.name, self.gender)
#
# s = Student("niefeng", "Male")
# s.__str__()
# print('Male' == Gender.Male)


# This file will aim to create the data for the RF model from Predicting Matchability - PM (2014) paper
# It will overwrite existing PM training data (db).
# Note: https://stackoverflow.com/questions/5189997/python-db-api-fetchone-vs-fetchmany-vs-fetchall
# Note: https://stackoverflow.com/questions/41464752/git-rebase-interactive-the-last-n-commits
# 03/11/2022: The official code (random forest) has been compiled,  g++ -O3 -DNDEBUG -Wall -march=native -o rforest rforest.cpp -I. -lboost_serialization -lboost_iostreams
# This file will also produce the training data for the rforest.cpp
# After this file runs, then compile the tool again: g++ -O3 -DNDEBUG -Wall -march=native -o rforest rforest.cpp -I. -lboost_serialization -lboost_iostreams
# just for sanity and train it: ./rforest -t 25 -d 25 -p pos.txt -n neg.txt -f rforest.gz

# To summarise:
# 1 - Run this file to generate the data for your sklearn model (.db) and the original tool rforest (.txt)
# 2 - Run "train_for_predicting_matchability.py"

# NOTE 14/02/2023:
# Removed the original tool as I got same results with python code.

import os
import sys
from database import COLMAPDatabase
from database import pair_id_to_image_ids, image_ids_to_pair_id
import numpy as np
from tqdm import tqdm
from parameters import Parameters
from query_image import get_descriptors

def get_matched_and_unmatched_descs(c, m, db, side=None):
    assert side != None
    pair_id = image_ids_to_pair_id(int(c), int(m))
    pair_data = db.execute("SELECT rows, cols, data FROM two_view_geometries WHERE pair_id = " + "'" + str(pair_id) + "'").fetchone()
    rows = pair_data[0]
    cols = pair_data[1]
    data = pair_data[2]
    zero_based_indices = COLMAPDatabase.blob_to_array(data, np.uint32).reshape([rows, cols])

    zero_based_matched_indices_left = zero_based_indices[:, 0]
    zero_based_matched_indices_right = zero_based_indices[:, 1]
    _, _, descs_left = get_descriptors(db, str(int(c)))
    _, _, descs_right = get_descriptors(db, str(int(m)))
    zero_based_unmatched_indices_left = np.delete(np.arange(descs_left.shape[0]), zero_based_matched_indices_left, axis=0)
    zero_based_unmatched_indices_right = np.delete(np.arange(descs_right.shape[0]), zero_based_matched_indices_right, axis=0)

    if(side=='left'): #central image
        matched = descs_left[zero_based_matched_indices_left]
        unmatched = descs_left[zero_based_unmatched_indices_left]
        matched = np.c_[matched, np.ones([matched[:, 0].shape[0], 1])]
        unmatched = np.c_[unmatched, np.zeros([unmatched[:, 0].shape[0], 1])]
        return matched, unmatched, zero_based_matched_indices_left, zero_based_unmatched_indices_left
    if (side == 'right'): #matched image or right image or m
        matched = descs_right[zero_based_matched_indices_right]
        unmatched = descs_right[zero_based_unmatched_indices_right]
        matched = np.c_[matched, np.ones([matched[:, 0].shape[0], 1])]
        unmatched = np.c_[unmatched, np.zeros([unmatched[:, 0].shape[0], 1])]
        return matched, unmatched, zero_based_matched_indices_right, zero_based_unmatched_indices_right

def gen_training_data(pair_ids, max_neighbours, max_pairs, descs_db):
    print("Getting Training Data..")
    image_ids = pair_id_to_image_ids(np.array(pair_ids))
    left_images_ids = image_ids[0] #can also use image_ids[1] here but it's more or less the same

    if(max_pairs == -1): #just use all UNIQUE pair ids
        max_pairs = len(np.unique(left_images_ids.flatten()))

    # TODO: max_pairs might be > than the number of unique left_images_ids images, so we need to check (rare case)
    rand_images = np.random.choice(np.unique(left_images_ids.flatten()), size=max_pairs, replace=False) #pick 500 random images (or central images) unique

    matched_images = {} #this will contain a random image id and its 13 * 2 (26),
    # This is where the "neighbours" are retrieved
    for rand_img_id in tqdm(rand_images):
        # get the neighbour pairs' image_ids indexes
        # to explain, we get the indexes of the random image id from the left/central image of the left and right images id list.
        # example: if rand_img_id == 4322, then we get the indexes of 4322 from left_images_ids or image_ids[0] and then use them to get
        # their individual pair(s) from image_ids[1]. we call those neighbours / matches_ids (below)
        # example: 4322 can be seen in index position 44 and 900. We pick the image_ids from the right images id list (image_ids[1]) at position 44, and 900.
        matches_ids = image_ids[1][np.where(left_images_ids == rand_img_id)[0]].flatten() #the images ids that are a match with rand_img_id
        # picking 13 neighbours here that are continuous makes no sense
        # so we pick 13 * 2 random images ids that are a match to rand_img_id
        if(len(matches_ids) > max_neighbours * 2):
            match_neighbours = np.random.choice(matches_ids, max_neighbours * 2, replace=False)
        else:
            match_neighbours = matches_ids #pick all, at this point it will be less than 26

        if(len(match_neighbours) == 0): #rand_img_id has no matches at all...sad..
            matched_images[rand_img_id] = 0
            continue

        assert (len(match_neighbours) != 0)
        matched_images[rand_img_id] = match_neighbours

    assert(len(matched_images) == max_pairs)

    print(f"Chose {len(matched_images.keys())} samples, or central images.")
    total_neighbours = 0
    all_ms = []
    for c, ms in tqdm(matched_images.items()):
        for m in ms:
            all_ms.append(m)
        total_neighbours += len(ms)
    print(f"An average of {int(total_neighbours/len(matched_images.keys()))} \"neighbours\" (both back and forward), or matched images, per central image (no: {len(matched_images)}).")
    print(f"A total of unique {np.unique(all_ms).shape[0]} \"neighbours\" (both back and forward), or matched images.")

    # At this point I will assign to each image id an array of positive / matched zero-based descs indices
    print("Getting the positive and negative desc indices for each image ..")
    # this contains all images, central and matches and their unique descs indexes
    tracked_indices = {}
    # Do central ones first
    for c, ms in tqdm(matched_images.items()):
        # for each c we only check with the first match, ms[0], can do other but same thing
        matched, unmatched, matched_indices, unmatched_indices = get_matched_and_unmatched_descs(c, ms[0], descs_db, side='left')
        assert (matched.shape[0] == matched_indices.shape[0])
        assert (unmatched.shape[0] == unmatched_indices.shape[0])
        tracked_indices[c] = matched_indices #c will always be unique as matched_images.keys() only contains unique ids

    assert (len(tracked_indices) == max_pairs)

    # Do matched ones second
    first_seen_matched_images = 0
    for c, ms in tqdm(matched_images.items()):
        for m in ms:
            # a right image (m) is an image that forms a pair with c (central image)
            # we have many c ones (unique) and many m ones. some c's can have the same m's.
            # so it is bound to happen that we will see some m again and again.
            # so make sure to keep their descs that were deemed positive once and not replace them with a negative one
            # as per the paper, "All points that appear in at least one match form the positive class"
            matched, unmatched, matched_indices, unmatched_indices = get_matched_and_unmatched_descs(c, m, descs_db, side='right')
            assert(matched.shape[0] == matched_indices.shape[0])
            assert(unmatched.shape[0] == unmatched_indices.shape[0])

            if(m in tracked_indices):
                # add new matched_indices and remove duplicates
                tracked_indices[m] = np.unique(np.concatenate((tracked_indices[m], matched_indices), 0))
            else:
                first_seen_matched_images +=1
                tracked_indices[m] = matched_indices

    # tracked_indices will contain less neighbours as it already contains central images ids (max_pairs should be the same length as centrals)
    assert (len(tracked_indices) == max_pairs + first_seen_matched_images)

    # at this point for each central and matched image we have the indices of the positive descs
    # we can easily get the negative ones

    # calculate the size of the descs array
    print("Calculating the size of, all_descs ..")
    decs_no = 0
    for image_id, matched_indices_array in tqdm(tracked_indices.items()):
        _, _, descs = get_descriptors(descs_db, str(int(image_id)))
        decs_no += descs.shape[0]
    all_descs = np.empty([decs_no, 129])

    # at this point get the descs using the indices and save them to all_descs
    print("Filling, all_descs ..")
    i = 0
    for image_id, matched_indices_array in tqdm(tracked_indices.items()):
        _, _, descs = get_descriptors(descs_db, str(int(image_id)))

        matched = descs[matched_indices_array]
        unmatched = np.delete(descs, matched_indices_array, axis=0)
        matched = np.c_[matched, np.ones([matched[:, 0].shape[0], 1])]
        unmatched = np.c_[unmatched, np.zeros([unmatched[:, 0].shape[0], 1])]

        assert((matched.shape[0] + unmatched.shape[0]) == descs.shape[0])

        for desc in matched:
            all_descs[i] = desc
            i += 1
        for desc in unmatched:
            all_descs[i] = desc
            i += 1

    assert i == all_descs.shape[0]
    return all_descs

def createDataForPredictingMatchabilityComparison(parameters, db_PM_path, max_neighbours=13, max_pairs=1200):
    db_live = COLMAPDatabase.connect(parameters.live_db_path)
    print("Selecting pair_ids..")
    pair_ids = db_live.execute("SELECT pair_id FROM two_view_geometries WHERE rows > 0 ORDER BY rows DESC").fetchall()

    print("Using pairs from openCV db..")
    openCV_db = COLMAPDatabase.connect(parameters.gt_db_path_mnm)

    print(f"Max pairs: {max_pairs} for this dataset..")

    # The pairs in the openCV gt db are less than the ones in the original gt db, this is because in create_universal_models.py, I pick pairs that only have
    # rows >0 from the gt db.
    # Now to make training fair I need to train on live data only, so I pick the live pairs from the openCV gt db.
    # The pairs in the openCV db are the same as in the original gt db image. Pair 12345 for example is the same in opencv gt and original live/gt db
    # This is because the opencv model was constructed from the original gt db

    # Choosing the pairs from the opencv gt db that are only in the live db, to avoid gt pairs in the training data
    pair_ids_opencv = openCV_db.execute("SELECT pair_id FROM two_view_geometries WHERE rows > 0 AND pair_id IN ({})".format(",".join(["?"] * len(pair_ids))), list(pair_id[0] for pair_id in pair_ids)).fetchall()
    pair_ids = pair_ids_opencv

    file_identifier = f"{max_pairs}_samples_opencv"
    training_data_db_path = os.path.join(db_PM_path, f"training_data_{file_identifier}.db")
    print(f"Creating training data db at {training_data_db_path}.. (will drop table if exists)")
    training_data_db = COLMAPDatabase.create_db_predicting_matchability_data(training_data_db_path)

    training_data_db.execute("BEGIN")
    training_descs = gen_training_data(pair_ids, max_neighbours, max_pairs, openCV_db) #need to pass openCV_db here as we will train on OpenCV descs

    print("Inserting data..")
    for training_desc in tqdm(training_descs):
        desc = training_desc[0:128].astype(np.uint8)
        matched = training_desc[128]
        training_data_db.execute("INSERT INTO data VALUES (?, ?)", (COLMAPDatabase.array_to_blob(desc),) + (matched,))

    print("Committing..")
    training_data_db.commit()
    print("Generating Stats..")
    stats = training_data_db.execute("SELECT * FROM data").fetchall()
    matched = training_data_db.execute("SELECT * FROM data WHERE matched = 1").fetchall()
    unmatched = training_data_db.execute("SELECT * FROM data WHERE matched = 0").fetchall()

    print("Total descs: " + str(len(stats)))
    print("Total matched descs: " + str(len(matched)))
    print("Total unmatched descs: " + str(len(unmatched)))
    print("% of matched decs: " + str(len(matched) * 100 / len(stats)))

    training_data_db.close()
    print("Done!")

root_path = "/media/iNicosiaData/engd_data/"
dataset = sys.argv[1] #HGE, CAB, LIN (or Other for CMU, retail shop)

# Note: Use parameters.predicting_matchability_comparison_data_lamar_no_samples max_pairs for HGE, CAB, LIN and 1200 for CMU

if(dataset == "HGE" or dataset == "CAB" or dataset == "LIN"):
    base_path = os.path.join(root_path, "lamar", f"{dataset}_colmap_model")
    print("Base path: " + base_path)
    parameters = Parameters(base_path)
    output_path = os.path.join(base_path, "predicting_matchability_comparison_data")
    os.makedirs(output_path, exist_ok=True)
    createDataForPredictingMatchabilityComparison(parameters, output_path, max_pairs=parameters.predicting_matchability_comparison_data_lamar_no_samples)

if(dataset == "CMU"):
    slices_names = ["slice2", "slice3", "slice4", "slice5", "slice6", "slice7", "slice8", "slice9", "slice10", "slice11", "slice12", "slice13", "slice14", "slice15",
                    "slice16", "slice17", "slice18", "slice19", "slice20", "slice21", "slice22", "slice23", "slice24", "slice25"]
    for slice_name in slices_names:
        base_path = os.path.join(root_path, "cmu", f"{slice_name}", "exmaps_data")
        print("Base path: " + base_path)
        parameters = Parameters(base_path)
        output_path = os.path.join(base_path, "predicting_matchability_comparison_data")
        os.makedirs(output_path, exist_ok=True)
        createDataForPredictingMatchabilityComparison(parameters, output_path)

if(dataset == "RetailShop"):
    base_path = os.path.join(root_path, "retail_shop", "slice1")
    print("Base path: " + base_path)
    parameters = Parameters(base_path)
    output_path = os.path.join(base_path, "predicting_matchability_comparison_data")
    os.makedirs(output_path, exist_ok=True)
    createDataForPredictingMatchabilityComparison(parameters, output_path)