import numpy as np
import scipy
from scipy import signal
import math
import matplotlib.pyplot as plt
from sympy import symbols, solve, erf
files=np.linspace(2,20,7,dtype=int)
d= np.loadtxt("en_All.csv",dtype=float,delimiter=',',skiprows=13)
c1=d[:,1]
c1=c1-np.mean(c1)
c2=d[:,2]
ENOB=8.8
h=6.63e-34
c=3e8
print('input impedence= ')
R=int(input())
if(R==50):
    BW1=0.1e9
else:
    BW1=0.1e9/R
BW2=0.1e9;
etac=0.9;
etad=0.9;
Gc=3.9e3;
Gd=39000;
wl=1550e-9;
r1=0.09
dV=0.08/2**ENOB
alphac=h*c*BW1*etac*Gc/wl;
alphad=h*c*BW2*etad*Gd/wl;
error=float(input('error probability= '))
t = symbols('t')
fun1 = 1-erf(t)-error
out=solve(fun1)
print(out)
mu_en,sigma_en=scipy.stats.norm.fit(c1)
lambda_bar=2**(0.5)*sigma_en*(out[0]);

l1=dV/(2*alphad)
l2=-(dV/(2*alphad))-1
pmax=np.zeros(len(files))
nrm=np.zeros(len(files))
ncm=np.zeros(len(files))
Mean_P=np.zeros(len(files))
Pguess=np.zeros(len(files))
f,psd_en=scipy.signal.welch(c1, fs=125e6, window='hann', nperseg=3000, noverlap=0.5, nfft=None, detrend='constant', return_onesided=True, scaling='density', axis=- 1, average='mean')
s= (7,48)
Var_P=np.zeros(s)
VarP=np.zeros(len(files))
stdP=np.zeros(len(files))

for i in files:
    print(i)
    a = np.loadtxt(str(i)+"_All.csv",dtype=float,delimiter=',',skiprows=13)
    b1=a[:,1]
    b2=a[:,2]
    b1=b1-np.mean(b1)
    b2=b2-np.mean(c2)
    ncm[int((i/2)-1)]=(np.mean(b2)-lambda_bar)/alphac;
    if ncm[int((i/2)-1)]<0:
        nrm[int((i/2)-1)]=0
        ncm[int((i/2)-1)]=0
    else:
        nrm[int((i/2)-1)]=4*ncm[int((i/2)-1)]*r1+4*r1**2-4*ncm[int((i/2)-1)]*r1**2-np.log10(0.5*error)+(-np.log10(0.5*error))**0.5*(8*ncm[int((i/2)-1)]*r1-np.log(0.5*error))**0.5/4*r1**2;

    f,psd_b=scipy.signal.welch(b1, fs=125e6, window='hann', nperseg=3000, noverlap=0.5, nfft=None, detrend='constant', return_onesided=True, scaling='density', axis=- 1, average='mean')

    for j in np.arange(1,48,1):
        Var_P[int((i/2)-1),j]=np.var(b1[int(np.ceil(len(b1)/1000)*(j-1)+1):int(np.ceil(len(b1)/1000)*j)])-np.var(c2[int(np.ceil(len(b1)/1000)*(j-1)+1):int(np.ceil(len(b1)/1000)*j)])
    VarP[int((i/2)-1)]=np.mean(Var_P[int((i/2)-1),0:48]);
    stdP[int((i/2)-1)]=np.std(Var_P[int((i/2)-1),0:48]);
    Mean_P[int((i/2)-1)]=np.mean(b2)
    if i==2:
        psd_ref=psd_b
        ref2=b2;
    SNL=(psd_b-psd_en)/(psd_ref-psd_en)
    SNL_the=np.zeros(len(f))+np.mean(b2)/np.mean(ref2)
    x_values=np.arange(np.min(b1),np.max(b1),(np.max(b1)-np.min(b1))/2**ENOB)
    mu,sigma=scipy.stats.norm.fit(b1)
    l3=(nrm[int((i/2)-1)]/4)**0.5;
    Pguess[int((i/2)-1)]=0.5*(erf(l1/l3)-erf(l2/l3));
    best_fit_line = scipy.stats.norm.pdf(x_values, mu, sigma)

    pmax[int((i/2)-1)]=np.max(best_fit_line)*(x_values[2]-x_values[1])
    plt.figure(2)
    plt.plot(x_values, best_fit_line,label="{:.3f} mW".format(np.mean(b2)))
    plt.legend(loc="best")
    plt.figure(1)
    plt.plot(f,10*np.log10(SNL))
    plt.plot(f,10*np.log10(SNL_the))
    plt.legend(["SNL", "SNL_the"])

    plt.xlim(0,50e6)
plt.savefig('SNL.png', dpi=600)
plt.figure(3)
plt.plot(Mean_P, -np.log2(Pguess),'o',label='$\\kappa$')
plt.plot(Mean_P, -np.log2(pmax),'+',label='$H_{min}(X|E)$')
plt.legend(loc="upper right")
plt.figure(4)
plt.plot(Mean_P, nrm,'o',label='$n_{rm}$')
plt.show()
A = np.polyfit(np.sort(Mean_P[1:8]),np.sort(VarP[1:8]), 1)
plt.figure(5)
plt.plot(np.sort(Mean_P[1:8]),A[0]*np.sort(Mean_P[1:8])+A[1])
plt.errorbar(np.sort(Mean_P[1:8]),np.sort(VarP[1:8]),yerr=np.sort(stdP[1:8]),xerr=None,fmt='o')


import requests
import json
from sys import argv
from os import mkdir
from bs4 import BeautifulSoup as soup

def clean_txt(txt):
  nv = txt.strip().replace('~', '...').replace('〜', '...').replace('～', '...').replace('+', '...').replace('・', ' / ').replace('」', ' ').replace('「',  '')
  return nv.replace('...', ' ... ')

def find_td(el, val, r=True):
  return el.find('td', f'jl-td-{val}', recursive=r)

type_key = argv[1] if len(argv) > 1 else 'nouns'
files = {
  'n1': [],
  'n2': [],
  'n3': [],
  'n4': [],
  'n5': [],
}

try:
  mkdir(f'./{type_key}')
except:
  pass

for n_key, n_lst in files.items():
  for idx in range(1, 11):
    proxy_lst = []
    page = soup(requests.get(f'https://jlptsensei.com/jlpt-{n_key}-{type_key}-vocabulary-list/page/{idx}').content, 'html.parser')
    for particle in page.find_all('tr', {'class': 'jl-row'}):
      hiragana = find_td(particle, 'vr').find('p').text
      romaji = find_td(particle, 'vr').text.replace(hiragana, '')
      info = find_td(particle, 'vm').text
      audio_name = f'ic_nrkt_{"".join(romaji.split(" "))}.mp3'
      sound = f'[sound:{audio_name}]'
      card = [clean_txt(val) for val in [hiragana, romaji, info, sound]]
      if '【' in card[1]:
        card[1] = card[1].split('【')[1].replace('】', '')
      proxy_lst.append(card)
    if len(proxy_lst) == 0:
      break
    else:
      n_lst.extend(proxy_lst)

for key, val in files.items():
  with open(f'./{type_key}/{key}.txt', 'w') as p_file:
    for card in val:
      p_file.write(" | ".join(card) + '\n')

with open(f'./{type_key}/full-list.json', 'w') as p_file:
  p_file.write(json.dumps(files))


import matplotlib.pyplot as plt
import numpy as np

plt.xlabel('x - axis')
plt.ylabel('y - axis')

v1 = [3, 0]
v2 = [1, 2]
vector_sum = np.add(v1, v2) # [4, 2]

x = [0, v1[0]]
y = [0, v1[1]]
x1 = [0, v2[0]]
y1 = [0, v2[1]]

plt.plot(x, y, x1, y1)
plt.fill_between([0, 1, 3, vector_sum[0], 3], [0, 2, 2, vector_sum[1], 0], color="red", alpha=0.4)

plt.show()
# plt.savefig("pgram2d.png")


import random
import itertools
import collections
import time

class Puzzle:
    """
    A class representing an '8-puzzle'.
    - 'board' should be a square list of lists
       e.g. [[1,2,3],[4,0,6],[7,5,8]]
    """
    def __init__(self, board,heuristic):
        self.width = len(board[0])
        self.board = board
        self.heuristic = heuristic

    @property
    def solved(self):
        """
        The puzzle is solved if the flattened board's numbers are in
        increasing order from left to right and the '0' tile is in the
        last position on the board
        """
        N = self.width * self.width
        return str(self) == '0' + ''.join(map(str, range(1,N)))

    @property
    def actions(self):
        """
        Return a list of 'move', 'action' pairs. 'move' can be called
        to return a new puzzle that results in sliding the '0' tile in
        the direction of 'action'.
        """
        def create_move(at, to):
            return lambda: self._move(at, to)

        moves = []
        for i, j in itertools.product(range(self.width),
                                      range(self.width)):
            direcs = {'R':(i, j-1),
                      'L':(i, j+1),
                      'D':(i-1, j),
                      'U':(i+1, j)}

            for action, (r, c) in direcs.items():
                if r >= 0 and c >= 0 and r < self.width and c < self.width and \
                   self.board[r][c] == 0:
                    move = create_move((i,j), (r,c)), action
                    moves.append(move)
        return moves

    @property
    def sum_not_placed(self):
      right_positions  = []
      for i in range(3):
        for j in range(3):
          right_positions.append((i,j))
      right_positions
      distance = 0
      for i in range(3):
          for j in range(3):
              if self.board[i][j] != 0:
                  if (i!=right_positions[self.board[i][j]][0]) or (j!=right_positions[self.board[i][j]][1]):
                    distance+=1
      return distance

    @property
    def sum_distance(self):
      right_positions  = []
      for i in range(3):
        for j in range(3):
          right_positions.append((i,j))
      right_positions
      distance = 0
      for i in range(3):
          for j in range(3):
              if self.board[i][j] != 0:
                  distance += abs(right_positions[self.board[i][j]][0] - i)+abs(right_positions[self.board[i][j]][1] - j)
      return distance

    @property
    def h(self):
      if self.heuristic == 'sum_distance':
        return self.sum_distance
      elif self.heuristic == 'sum_not_placed':
        return self.sum_not_placed
      else:
        raise Exception('heuristic not knonw')

    def copy(self):
        """
        Return a new puzzle with the same board as 'self'
        """
        board = []
        for row in self.board:
            board.append([x for x in row])
        return Puzzle(board,self.heuristic)

    def _move(self, at, to):
        """
        Return a new puzzle where 'at' and 'to' tiles have been swapped.
        NOTE: all moves should be 'actions' that have been executed
        """
        copy = self.copy()
        i, j = at
        r, c = to
        copy.board[i][j], copy.board[r][c] = copy.board[r][c], copy.board[i][j]
        return copy

    def pprint(self):
        for row in self.board:
            print(row)
        print()

    def shuffle(self,n):
      "shuffles the puzzle n times randomly"
      puzzle = self
      for _ in range(n):
          puzzle = random.choice(puzzle.actions)[0]()
      return puzzle

    def __str__(self):
        return ''.join(map(str, self))

    def __iter__(self):
        for row in self.board:
            yield from row


import numpy as np
import cv2
import os
from matplotlib import pyplot as plt
from imutils import resize
from filelist import FileList  # import the filelist class
import logging
import argparse

# # for getting focal length from the photos database
from issimage import ISSIMAGE

# filename = r'C:\Users\lrmayer\Documents\Mayer\MyOpenCV_Code\Smallcoins.jpg'
# filename = r'F:\imagews\training\Earthlimb_ISS005-E-8536.jpg'  # perfect for learning
#filename = r'F:\imagews\training\Earthlimb_ISS009-E-23413.jpg'
#filename = r'F:\imagews\training\Earthlimb_ISS009-E-28843.jpg'
# filename = r'F:\imagews\training\Land_ISS009-E-22319.jpg'  # dark part is window, not limb
# filename = r'F:\imagews\training\Night_ISS028-E-26603.jpg'

# python find_limb_part2.py E:\BetterLightning --log=debug --list="E:\BetterLightning\lightning.txt"
# python find_limb_part2.py F:\imagews\training --log=debug --list="F:\imagews\training\earthlimb.txt"
# python find_limb_part2.py F:\imagews\training --log=debug --list="F:\imagews\training\various.txt"
# python find_limb_part2.py F:\imagews\training --log=debug --list="F:\imagews\training\aurora.txt"
# python find_limb_part2.py F:\imagews\training --log=debug --list="F:\imagews\training\list.txt"

# -------------------------------------------
# Thresholding for finding where the limb is
# -------------------------------------------
doThreshold = False
threshold = 50

# -------------------------------------------------
# Get rid of small bright features (stars,  noise)
# -------------------------------------------------
doOpening = True

# -------------------------------------------------
# Get rid of small bright features (stars,  noise)
# -------------------------------------------------
doClosing = True

# -----------------------------------------------------
# Construct the argument parse and parse the arguments
# -----------------------------------------------------
ap = argparse.ArgumentParser()
ap = argparse.ArgumentParser(description="Find earth limb in images")
ap.add_argument("dataset", help="Path to images")  # this is required

# Add logging file argument
ap.add_argument("-l", "--log", help="Logging Value", default="info")

# Add list file argument
ap.add_argument("-li", "--list", help="List of images to read")

# parse the arguments
args = ap.parse_args()

# -------------------------------------------------------------
# Set up logging file
# -------------------------------------------------------------
# loglevel is bound to the string value obtained from the
# command line argument. Convert to upper case to allow the user to
# specify --log=DEBUG or --log=debug
# -------------------------------------------------------------
numeric_level = getattr(logging, args.log.upper(), None)
if not isinstance(numeric_level, int):
    raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level, filename='find_limb_part2.log', filemode='w')

# -------------------------------------
# Get the list of image names to read
# -------------------------------------
theList = FileList(args.dataset,  args.list)
imagePaths = theList.getPathFilenames()

logging.info("imagePath is %s", imagePaths)

# --------------------------------------
# Loop over the input dataset of images
# --------------------------------------
for filename in imagePaths:

    if args.list :
        if not os.path.isfile(filename) :
               sys.exit("filename " + filename + "  does not exist")

        # Read image
        img = cv2.imread(filename)

        # resize the image
        # def resize(image, width = None, height = None, inter = cv2.INTER_AREA)
        resized = resize(img, width=500)

        # initialize ISSIMAGE class for setting up the image
        xx = ISSIMAGE(filename)

        xx.resize()

        xx.show()

        # ----------------------
        # Get the sun elevation
        # ----------------------
        sunElev = xx.get_sun_elev()
        logging.debug("Sun elevation = %s", sunElev)

        # ----------------------
        # Get the focal length
        # ----------------------
        focalLength = xx.get_focal_length()
        logging.debug("Focal length = %s", focalLength)

        # -----------------
        # convert to gray
        # -----------------
        gray = cv2.cvtColor(resized,cv2.COLOR_BGR2GRAY)

        # Do histogram equalization ??? (only needed for night time limb)
        # equalize the histogram of the image
        eq = cv2.equalizeHist(gray)
        cv2.imshow("histogram of open/close", eq)
        #closed = eq.copy()

        # ------------------------------------------------------
        # Smooth the image
        # nn = the neighborhood size
        # For finding the limb, want the image to be very smooth
        # -------------------------------------------------------
        nn = 11  # Get less edges w/ higher n
        #nn = 5
        #gray = cv2.GaussianBlur(gray, (nn, nn), 0)

        # -----------------------------------------------
        # Bilateral filtering preserves the edges
        # 2nd parameter is the window size for smoothing
        # -----------------------------------------------
        #filtered = cv2.bilateralFilter(gray,9,75,75)
        filtered = cv2.bilateralFilter(gray,19,75,75)

        cv2.imshow("bilateral filter", filtered)

        # ==================================================================
        # Do an opening to get rid of more noise (the small bright areas)
        # ==================================================================
        if (doOpening):
            #kernel = np.ones((3, 3), np.uint8)
            kernel = np.ones((9, 9), np.uint8)
            #kernel = np.ones((19, 19), np.uint8)  # to big for night images
            opened = cv2.morphologyEx(filtered, cv2.MORPH_OPEN, kernel)
        else:
            opened = filtered.copy()

        # ---------------------------------------
        # Do closing to get of small dark spots
        # ---------------------------------------
        if (doClosing):
            #kernel = np.ones((3, 3), np.uint8)
            #kernel = np.ones((9, 9), np.uint8)
            #kernel = np.ones((19, 19), np.uint8)
            closed = cv2.morphologyEx(opened, cv2.MORPH_CLOSE, kernel)
        else:
            closed = opened.copy()

        cv2.imshow("opened/closed", closed)

        # ---------------------------------------------------------------------------------------------
        # Show the gradient image (for testing whether this is a viable method for getting night limb)
        # ---------------------------------------------------------------------------------------------
        kernel = np.ones((3, 3), np.uint8)
        gradient = cv2.morphologyEx(closed, cv2.MORPH_GRADIENT, kernel)
        cv2.imshow("gradient of opened/closed", gradient)

        # ---------------------
        # Threshold the image
        # ---------------------
        if (doThreshold) :

            if (sunElev > 10) :
                # image needs to be 1D for threshold function
                # Simple, binary thresholding
                #ret, thresh = cv2.threshold(opened, 100, 255, cv2.THRESH_BINARY)  # not bad, but may be too high
                # pretty good, but not good w/ very bright limb images : Earthlimb_ISS009-E-16194.jpg - Earthlimb_ISS009-E-16199.jpg
                # May need to figure out peaks of bimodal color histogram on the gray image or something like that

                logging.debug("threshold = %s", threshold)
                ret, thresh = cv2.threshold(closed, threshold, 255, cv2.THRESH_BINARY)

            elif (sunElev < -30. and sunElev > -70.) :

                #threshold = 75  # to get the bright limb  # too low
                #threshold = 100  # to get the bright limb # too high
                #threshold = 90  # too high
                #threshold = 85 # too high
                threshold = 80
                #logging.debug("threshold = %s", threshold)
                # this global thresholding not really working b/c of variable lighting
                #ret, thresh = cv2.threshold(closed, threshold, 255, cv2.THRESH_BINARY)

                # adaptive gaussian thresholding
                logging.debug("Adaptive Gaussian threshold")
                nn = 11  # not good - too many, lots of them dots
                nn = 5  # better, but only gets the really big contrasts
                nn = 3  # same as above
                #nn = 9  # not sure why there is a bunch of dot contours above limb
                #thresh = cv2.adaptiveThreshold(closed, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, nn, 2)
                thresh = cv2.adaptiveThreshold(closed, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, nn, 10)
                #  This only gets the egdes that are very big in contrast (very dark next to very light)
                #thresh = cv2.adaptiveThreshold(closed, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, nn, -10)
                #ret, thresh = cv2.threshold(closed, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\

            else:

                logging.debug("binary + otsu threshold")
                ret, thresh = cv2.threshold(closed,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)

        else:
            thresh = opened.copy()   # gives too many contours

        #cv2.imshow("threshold", thresh)

        # ---------------------------------------------------------------
        # compute edges on the smoothed image
        # any gradient values below 30 are considered
        # non-edges whereas any values above 150 are considered
        # sure edges.
        # ----------------------------------------------------------------
        edgeThresh1 = 30
        edgeThresh2 = 50  # use for thresholded day limb

        # too many contours when sun elev is high (by night standards)
        # no contours in very low sun elev : -76
        edgeThresh1 = 50
        edgeThresh2 = 100  # alot of contours, but maybe its ok, as long as it gets the limb

        #edgeThresh2 = 200  # not enough
        #edgeThresh2 = 150  # not enough
        #edgeThresh2 = 150  # good for city lights

        edged = cv2.Canny(thresh, edgeThresh1, edgeThresh2)  # use for day limb

        # run the canny edge detection on smoothed image
        #                image, threshold1, threshold2
        # Any gradient value larger than threshold2 is considered
        # to be an edge. Any value below threshold1 is considered
        # not to be an edge. Values in between threshold1
        # and threshold2 are either classified as edges or non-edges
        # based on how their intensities are 'connected'
        # threshold1 = 30
        # threshold2 = 150
        # canny = cv2.Canny(image, threshold1, threshold2)
        # cv2.imshow("Canny 1", canny)
        # cv2.waitKey(0)

        cv2.imshow("Edges", edged)

        # find the contours of the edges. cnts is the returned contours
        # This method returns
        # a 3-tuple of: (1) our image after applying contour detection
        # (which is modified and essentially destroyed), (2) the
        # contours themselves, cnts, and (3) the hierarchy of the contours
        # (OpenCV 3.0 is different from OpenCV 2.4, which returns a tuple of 2)
        # This function is destructive
        # to the image you pass in. If you intend using that image
        # later on in your code, it's best to make a copy of it, using
        # the NumPy copy method.
        # The second argument is the type of contours we want.
        # We use cv2.RETR_EXTERNAL to retrieve only the outermost
        # contours (i.e., the contours that follow the outline of the
        # coin). We can also pass in cv2.RETR_LIST to grab all contours.
        # Other methods include hierarchical contours using
        # cv2.RETR_COMP and cv2.RETR_TREE, but hierarchical contours
        # are outside the scope of this book.
        # of resources.
        # Our contours cnts is simply a Python list. We can use
        # the len function on it to count the number of contours that
        # were returned.
        (_, cnts, _) = cv2.findContours(edged.copy(), cv2.RETR_EXTERNAL,
            cv2.CHAIN_APPROX_SIMPLE)

        # draw the contours in green on the coins image
        # contour index of -1 means draw all contours.  If just
        # wanted to draw ith contour, just enter i
        # The last argument is the thickness of the line we
        # are drawing. We'll draw the contour with a thickness of
        # two pixels.
        # cv2.drawContours(resized, cnts, -1, (0, 255, 0), 2)
        # cv2.imshow("contours", resized)

        # See :
        # http://docs.opencv.org/3.1.0/dd/d49/tutorial_py_contour_features.html
        # for information about contour characterics (used below, like getting bounding rectangle)

        green = (0, 255, 0)
        red = (0, 0, 255)
        yellow = (0, 255, 255)

        # write number of contours on the image
        numContours = len(cnts)

        # loop over contours
        for (i, c) in enumerate(cnts):

            #logging.info("\nContour #%s",format(c))
            logging.info("\nContour #%s", c)

            # -------------------------------------------------------------------------------------------
            # Straight Bounding Rectangle :
            # A straight rectangle, it
            # doesn't consider the rotation of the object. So area of the bounding rectangle won't be
            # minimum. It is found by the function sget the bounding rectangle
            # -------------------------------------------------------------------------------------------
            (x, y, w, h) = cv2.boundingRect(c)
            #print("x,y,w,h : {} {} {} {} ".format(x,y,w,h))

            # get the aspect ratio
            aspect_ratio = float(w) / float(h)
            #print("aspect_ratio : {}".format(aspect_ratio))
            logging.info("aspect ratio : %12.5f",aspect_ratio)

            # get extent : Extent is the ratio of contour area to bounding rectangle area.
            # extent = object area/bounding rectangle area
            area = cv2.contourArea(c)  # get the contour area
            logging.info("contour area : %12.5f", area)
            rect_area = w * h
            extent = float(area) / rect_area
            logging.info("extent : %12.5f", extent)

            # ------------------------------------------------------------------------------------
            # Get the minimum closing circle that completely covers the object with minimum area.
            # ------------------------------------------------------------------------------------
            (x, y), radius = cv2.minEnclosingCircle(cnts[i])
            center = (int(x), int(y))
            radius = int(radius)
            #cv2.circle(resized, center, radius, (0, 255, 255), 2)
            logging.info("x,y, radius of min circle : %s %s %12.3f", x, y, radius)

            # Rotated Rectangle :
            # The bounding rectangle is drawn
            # with minimum area, so it considers the rotation also.  The function used is cv2.minAreaRect().
            # It returns a Box2D structure which contains following detals - ( center (x, y), (width, height),
            # angle of rotation ).But to draw this rectangle, we need 4 corners of the rectangle.
            # It is obtained by the function cv2.boxPoints()

            rect = cv2.minAreaRect(cnts[i])
            box = cv2.boxPoints(rect)
            box = np.int0(box)
            cv2.drawContours(resized, [box], 0, (0, 255, 255), 2)

            if  (area < 1.) :
            #if (area == 0.):
                cv2.drawContours(resized, cnts, i, red, 2)
            else:
                cv2.drawContours(resized, cnts, i, green, 2)

            logging.info("-----------------------")

        cv2.putText(resized, str(numContours) + ' contours', (20, 60),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1)

        cv2.imshow("contours", resized)
        cv2.waitKey(0)

# # noise removal
# kernel = np.ones((3,3),np.uint8)
# opening = cv2.morphologyEx(thresh,cv2.MORPH_OPEN,kernel, iterations = 2)
#
# # sure background area
# sure_bg = cv2.dilate(opening,kernel,iterations=3)
#
# # Finding sure foreground area
# dist_transform = cv2.distanceTransform(opening,cv2.DIST_L2,5)
# ret, sure_fg = cv2.threshold(dist_transform,0.7*dist_transform.max(),255,0)
#
# # Finding unknown region
# sure_fg = np.uint8(sure_fg)
# unknown = cv2.subtract(sure_bg,sure_fg)
#
# # Marker labelling
# ret, markers = cv2.connectedComponents(sure_fg)
#
# # Add one to all labels so that sure background is not 0, but 1
# markers = markers+1
#
# # Now, mark the region of unknown with zero
# markers[unknown==255] = 0
#
# #markers = cv2.watershed(img,markers)
# #img[markers == -1] = [255,0,0]
#
# markers = cv2.watershed(resized,markers)
# resized[markers == -1] = [255,0,0]

#cv2.imshow("watershed", resized)
#cv2.waitKey(0)


"""
Different Methods to download data
"""
import logging
import os

import pandas as pd
from ..processing.workspace import DirectoryOperations
from datetime import datetime

logger = logging.getLogger(__name__)

class SaveData:
    @classmethod
    def write_file(cls, df: pd.DataFrame, tag: str,
                   date_tag: str, path: str = r'N:\DSI\ASI4\ASI42\Partilha\Data\sources\clean_data',
                   product: str = "master",
                   delimiter: str = ',', encoding: str = 'UTF-8-SIG', file_type: str = 'csv',
                   date_part: str = '') -> None:
        """
        Function to save file in csv
        :param date_part:
        :param file_type:
        :param df:
        :param path:
        :param tag:
        :param date_tag:
        :param delimiter:
        :param encoding:
        :param product:
        :return:
        """
        # Saving clean source
        logger.info('Saving file')
        assert type(df) == pd.DataFrame, "The df parameter must be a DataFrame"
        assert type(tag) == str, "The tag parameter must be a string"
        assert type(date_tag) == str, "The date_tag parameter must be a string"
        assert type(path) == str, "The path must be a string"
        assert type(delimiter) == str, "The delimiter must be a string"
        assert type(encoding) == str, "The encoding must be a string"
        assert type(file_type) == str, "The file_type must be a string"
        assert type(date_part) == str,  "The date_part must be a string"

        # Check directory path with product
        DirectoryOperations.check_dir(path, product)

        if date_part != '':
            complete_path = os.path.join(path, product, tag)
            DirectoryOperations.check_dir(complete_path, '{}_{}'.format(tag, date_part))
            complete_path = os.path.join(complete_path, '{}_{}'.format(tag, date_part))
            tag = '{}_{}'.format(tag, date_part)
        else:
            # Check directory path with product and source tag
            complete_path = os.path.join(path, product)
            DirectoryOperations.check_dir(complete_path, tag)
            complete_path = os.path.join(complete_path, tag)
        # Check date_tag
        try:
            datetime.strptime(date_tag, "%Y%m%d_%H%M%S")
        except ValueError as e:
            logger.error(e)
            raise ValueError("The given str_time must be on the format: {0}".format("%Y%m%d_%H%M%S"))

        df.to_csv(
            os.path.join(complete_path,
                         tag + '_' + date_tag + '.' + file_type),
            sep=delimiter,
            index=False,
            encoding=encoding)

        logger.info("Source updated")

    @classmethod
    def write_table(cls, df: pd.DataFrame, tag: str, date_tag: str) -> None:
        """
        Function to write a table
        :param df:
        :param tag:
        :param date_tag:
        :return:
        """
        assert type(df) == pd.DataFrame, "The df parameter must be a DataFrame"
        assert type(tag) == str, "The df parameter must be a string"
        assert type(date_tag) == str, "The df parameter must be a string"


# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
This module provides classes to perform grouping of stars.
"""

import numpy as np

__all__ = ['SourceGrouper']

class SourceGrouper:
    """
    Class to group sources into clusters based on a minimum separation
    distance.

    This class uses the `"Density-based Spatial Clustering
    of Applications with Noise" clustering algorithm
    <https://scikit-learn.org/stable/modules/clustering.html#dbscan>`_
    from `scikit-learn <https://scikit-learn.org/>`_.

    Parameters
    ----------
    min_separation : float
        The minimum distance (in pixels) such that any two sources
        separated by less than this distance will be placed in the same
        group if the ``min_size`` criteria is also met.

    min_size : int, optional
        The minimum number of sources necessary to form a group.

    metric : str or callable (default='euclidean'), optional
        The metric to use when calculating distance between each pair of
        sources.

    algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
        The algorithm to use to find nearest neighbors.

    leaf_size : int, optional (default = 30)
        The leaf size passed to the BallTree or cKDTree function.
        Changing ``leaf_size`` will not affect the results of a query,
        but can significantly impact the speed of a query and the memory
        required to store the constructed tree.

    References
    ----------
    [1] scikit-learn DBSCAN.
        https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html

    Notes
    -----
    ``min_separation`` corresponds to ``eps`` in `sklearn.cluster.DBSCAN
    <https://scikit-learn.org/stable/modules/generated/sklearn.cluster.D
    BSCAN.html>`_.
    """
    def __init__(self, min_separation, *, min_size=1, metric='euclidean',
                 algorithm='auto', leaf_size=30):
        self.min_separation = min_separation
        self.min_size = min_size
        self.metric = metric
        self.algorithm = algorithm
        self.leaf_size = leaf_size

    def __call__(self, x, y):
        """
        Group sources into clusters based on a minimum distance
        criteria.

        Parameters
        ----------
        x, y : 1D float `~numpy.ndarray`
            The 1D arrays of the x and y centroid coordinates of the
            sources.

        Returns
        -------
        result : 1D int `~numpy.ndarray`
            A 1D array of the groups, in the same order as the input x
            and y coordinates.
        """
        return self._group_sources(x, y)

    def _group_sources(self, x, y):
        """
        Group sources into clusters based on a minimum distance
        criteria.

        Parameters
        ----------
        x, y : 1D float `~numpy.ndarray`
            The 1D arrays of the x and y centroid coordinates of the
            sources.

        Returns
        -------
        result : 1D int `~numpy.ndarray`
            A 1D array of the groups, in the same order as the input x
            and y coordinates.
        """
        from sklearn.cluster import DBSCAN

        xypos = np.transpose((x, y))
        dbscan = DBSCAN(eps=self.min_separation, min_samples=self.min_size,
                        metric=self.metric, algorithm=self.algorithm,
                        leaf_size=self.leaf_size)
        group_id = dbscan.fit(xypos).labels_ + 1

        return group_id


# dataset name: XYGraphP1

from numpy import ediff1d
from models.sage_neighsampler import SAGE_NeighSamplerEnsemble
from models.trans_neighsampler_v3 import TransNetNSV3
from utils import XYGraphP1
from utils.utils import prepare_folder
from utils.evaluator import Evaluator
from torch_geometric.loader import NeighborSampler
# from torch_geometric.loader import ImbalancedSampler, NeighborLoader
from models import SAGE_NeighSampler, GAT_NeighSampler, GATv2_NeighSampler, GCN2ConvEnsemble, GCN2Net, TransNetNS, TransNetNSV2, PNANetNS
from tqdm import tqdm

import argparse

import torch
import torch.nn.functional as F
import torch.nn as nn

import torch_geometric.transforms as T
from torch_sparse import SparseTensor
from torch_geometric.utils import to_undirected
import pandas as pd

from sklearn.metrics import confusion_matrix
import numpy as np

class FocalLoss(nn.Module):

    def __init__(self, gamma=10.0, alpha=0.99, size_average=False):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.alpha = alpha
        if isinstance(alpha, (float, int)): self.alpha = torch.Tensor([alpha, 1 - alpha])
        if isinstance(alpha, list): self.alpha = torch.Tensor(alpha)
        self.size_average = size_average

    def forward(self, input, target):
        if input.dim()>2:
            input = input.view(input.size(0), input.size(1), -1)  # N,C,H,W => N,C,H*W
            input = input.transpose(1, 2)                         # N,C,H*W => N,H*W,C
            input = input.contiguous().view(-1, input.size(2))    # N,H*W,C => N*H*W,C
        target = target.view(-1, 1)

        # logpt = F.log_softmax(input, dim=1)
        logpt = input
        # 获取target对应的0/1分类置信度
        logpt = logpt.gather(1, target)
        logpt = logpt.view(-1)
        pt = logpt.exp()

        if self.alpha is not None:
            if self.alpha.type() != input.data.type():
                self.alpha = self.alpha.type_as(input.data)
            at = self.alpha.gather(0, target.data.view(-1))
            logpt = logpt * at

        loss = -1 * (1 - pt)**self.gamma * logpt
        if self.size_average: return loss.mean()
        else: return loss.sum()

eval_metric = 'auc'

sage_neighsampler_parameters = {'lr':0.003
              , 'num_layers':2
              , 'hidden_channels':128
              , 'dropout':0.
              , 'batchnorm': False
              , 'l2':5e-7
             }

gat_neighsampler_parameters = {'lr':0.003
              , 'num_layers':2
              , 'hidden_channels':128
              , 'dropout':0.0
              , 'batchnorm': False
              , 'l2':5e-7
             , 'layer_heads':[4,1]
             }

gatv2_neighsampler_parameters = {'lr':0.003
              , 'num_layers':2
              , 'hidden_channels':128
              , 'dropout':0.1
              , 'batchnorm': False
              , 'l2':5e-6
             , 'layer_heads':[4,4]
             }

gcn2ensemble_parameters = {'lr':1e-3
              , 'num_layers':3
              , 'hidden_channels':128
              , 'dropout':0
              , 'batchnorm': False
              , 'l2':5e-7
             }
gcn2_parameters = {'lr':1e-2
              , 'num_layers':2
              , 'hidden_channels':128
              , 'dropout':0
              , 'batchnorm': False
              , 'l2':5e-7
             }

trans_parameters = {'lr':1e-3
              , 'num_layers':3
              , 'hidden_channels':64
              , 'dropout':0.3
              , 'batchnorm': False
              , 'l2':5e-4
              , 'heads':7
             }

transv2_parameters = {'lr':1e-3
              , 'num_layers':2
              , 'hidden_channels':64
              , 'dropout':0.3
              , 'batchnorm': False
              , 'l2':5e-4
              , 'heads':30
             }

transv3_parameters = {'lr':5e-4
              , 'num_layers':3
              , 'hidden_channels':32
              , 'dropout':0.3
              , 'batchnorm': False
              , 'l2':5e-4
              , 'heads':7
             }

pna_parameters = {'lr':1e-3
              , 'num_layers':3
              , 'hidden_channels':128
              , 'dropout':0.1
              , 'batchnorm': False
              , 'l2':5e-4
              , 'heads':4
             }

def train_nolabel(epoch, train_loader, model, data, train_idx, optimizer, device, no_conv=False):
    model.train()

    pbar = tqdm(total=train_idx.size(0), ncols=80)
    pbar.set_description(f'Epoch {epoch:02d}')

    total_loss = total_correct = 0
    for batch_size, n_id, adjs in train_loader:
        # `adjs` holds a list of `(edge_index, e_id, size)` tuples.
        adjs = [(edge_index.to(device), data.edge_attr[e_id], size) for (edge_index, e_id, size) in adjs]

        optimizer.zero_grad()
        # noise = torch.normal(0, 1, size=data.x[n_id].shape).to(device)
        out = model(data.x[n_id], adjs)

        tmp_y = out.argmax(1)

        alpha = 0.001
        # loss = F.nll_loss(out[tmp_y <= 1], tmp_y[tmp_y <= 1])
        loss = alpha*F.nll_loss(out[tmp_y <= 1], tmp_y[tmp_y <= 1], weight=torch.tensor([0.01, 0.99]).to(device))
        # loss = torch.log(1. + F.nll_loss(out, data.y[n_id[:batch_size]]))
        # loss = F.nll_loss(out, data.y[n_id[:batch_size]], weight=torch.tensor([1., 20.]).cuda())
        # loss = FocalLoss(gamma=10.0, alpha=0.01)(out[tmp_y <= 1], tmp_y[tmp_y <= 1])
        loss.backward()
        optimizer.step()

        total_loss += float(loss)
        # total_loss += float(F.nll_loss(out, data.y[n_id[:batch_size]]))
        pbar.update(batch_size)

    pbar.close()
    loss = total_loss / len(train_loader)

    return loss

def train(epoch, train_loader, model, data, train_idx, optimizer, device, no_conv=False):
    model.train()

    pbar = tqdm(total=train_idx.size(0), ncols=80)
    pbar.set_description(f'Epoch {epoch:02d}')

    total_loss = total_correct = 0
    for batch_size, n_id, adjs in train_loader:
        # `adjs` holds a list of `(edge_index, e_id, size)` tuples.
        adjs = [(edge_index.to(device), data.edge_attr[e_id], size) for (edge_index, e_id, size) in adjs]

        optimizer.zero_grad()
        # noise = torch.normal(0, 1, size=data.x[n_id].shape).to(device)
        out = model(data.x[n_id], adjs)
        tmp_y = data.y[n_id[:batch_size]]

        # loss = F.nll_loss(out[tmp_y <= 1], tmp_y[tmp_y <= 1])
        loss = F.nll_loss(out[tmp_y <= 1], tmp_y[tmp_y <= 1], weight=torch.tensor([0.01, 0.99]).to(device))
        # loss = torch.log(1. + F.nll_loss(out, data.y[n_id[:batch_size]]))
        # loss = F.nll_loss(out, data.y[n_id[:batch_size]], weight=torch.tensor([1., 20.]).cuda())
        # loss = FocalLoss(gamma=10.0, alpha=0.01)(out[tmp_y <= 1], tmp_y[tmp_y <= 1])
        loss.backward()
        optimizer.step()

        total_loss += float(loss)
        # total_loss += float(F.nll_loss(out, data.y[n_id[:batch_size]]))
        pbar.update(batch_size)

    pbar.close()
    loss = total_loss / len(train_loader)

    return loss

@torch.no_grad()
def test(layer_loader, model, data, split_idx, device, no_conv=False):
    # data.y is labels of shape (N, )
    model.eval()

    out = model.inference(data.x, layer_loader, device, data)
#     out = model.inference_all(data)
    y_pred = out.exp()  # (N,num_classes)

    losses = dict()
    for key in ['train', 'valid', 'test']:
        node_id = split_idx[key]
        node_id = node_id.to(device)
        losses[key] = F.nll_loss(out[node_id], data.y[node_id], weight=torch.tensor([0.1, 0.9]).to(device)).item()
        # losses[key] = FocalLoss()(out[node_id], data.y[node_id]).item()

        if key == 'valid':
            # print(out[node_id].argmax(1))
            print(confusion_matrix(data.y[node_id].cpu().numpy(), out[node_id].argmax(1).cpu().numpy()))

    return losses, y_pred

@torch.no_grad()
def test_v2(train_loader, val_loader, test_loader, model, data, split_idx, device, no_conv=False):
    # data.y is labels of shape (N, )
    model.eval()

    out = torch.zeros((data.y.shape[0], 2)).to(device)

    for batch_size, n_id, adjs in train_loader:
        adjs = [(edge_index.to(device), data.edge_attr[e_id], size) for (edge_index, e_id, size) in adjs]
        out[n_id[:batch_size]] = model(data.x[n_id], adjs)

    for batch_size, n_id, adjs in val_loader:
        adjs = [(edge_index.to(device), data.edge_attr[e_id], size) for (edge_index, e_id, size) in adjs]
        out[n_id[:batch_size]] = model(data.x[n_id], adjs)

    for batch_size, n_id, adjs in test_loader:
        adjs = [(edge_index.to(device), data.edge_attr[e_id], size) for (edge_index, e_id, size) in adjs]
        out[n_id[:batch_size]] = model(data.x[n_id], adjs)

    losses = dict()
    for key in ['train', 'valid', 'test']:
        node_id = split_idx[key]
        node_id = node_id.to(device)
        tmp_y = data.y[node_id]
        # losses[key] = F.nll_loss(out[node_id], data.y[node_id], weight=torch.tensor([0.01, 0.99]).to(device)).item()
        # loss = FocalLoss(gamma=2.0, alpha=0.01)(out[tmp_y <= 1], tmp_y[tmp_y <= 1])
        if key in ['train', 'valid']:
            losses[key] = F.nll_loss(out[node_id][tmp_y<=1], data.y[node_id][tmp_y<=1], weight=torch.tensor([0.1, 0.9]).to(device))
            # losses[key] = FocalLoss(gamma=2.0, alpha=0.01)(out[node_id][tmp_y<=1], data.y[node_id][tmp_y<=1])
        else:
            losses[key] = 0.
        if key == 'valid':
            # print(out[node_id].argmax(1))
            print(confusion_matrix(data.y[node_id].cpu().numpy(), out[node_id].argmax(1).cpu().numpy()))

    y_pred = out.exp()  # (N,num_classes)
    return losses, y_pred

def main():
    parser = argparse.ArgumentParser(description='minibatch_gnn_models')
    parser.add_argument('--device', type=int, default=0)
    parser.add_argument('--dataset', type=str, default='XYGraphP1')
    parser.add_argument('--log_steps', type=int, default=10)
    parser.add_argument('--model', type=str, default='mlp')
    parser.add_argument('--epochs', type=int, default=100)

    args = parser.parse_args()
    print(args)

    no_conv = False
    if args.model in ['mlp']: no_conv = True

    device = f'cuda:{args.device}' if torch.cuda.is_available() else 'cpu'
    device = torch.device(device)
    # T.ToUndirected(),
    dataset = XYGraphP1(root='./', name='xydata', transform=T.Compose([T.ToUndirected(), T.ToSparseTensor()]))

    nlabels = dataset.num_classes
    if args.dataset =='XYGraphP1': nlabels = 2

    data = dataset[0]

    # data.x = data.x[:,:2]
    # data.x = data.x.index_select(1, torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16]))
    # for i in range(data.x.shape[-1]):
    #     tmp = data.x[:, i]
    #     tmp[tmp == -1] = 0
    #     data.x[:, i] = tmp
    # data.x = data.x[,]
    # data.adj_t = data.adj_t.to_symmetric()
    # data.x = torch.cat([torch.tensor(np.load('./fixed_features.npy')), data.x[:, -1].unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/type2.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/type3.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/in_degree.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/out_degree.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/all_degree.npy')).unsqueeze(1)], dim=-1)

    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/max_contact_degree_all.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/max_contact_degree_in.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/max_contact_degree_out.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/min_contact_degree_all.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/min_contact_degree_in.npy')).unsqueeze(1)], dim=-1)
    data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/min_contact_degree_out.npy')).unsqueeze(1)], dim=-1)
    # data.x = torch.cat([data.x, (data.y == 2).float(), (data.y == 3).float()], dim=-1)

    if args.dataset in ['XYGraphP1']:
        x = data.x
        x = (x-x.mean(0))/(x.std(0) + 1e-40)
        data.x = x

    # data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/act_behavior.npy')).float()], dim=-1)

    use_edge_vec = True
    use_edge_ts = True
    use_back_nodes = True
    if use_edge_vec:
        # edge_vec = np.load("./edge_vec.npy")
        edge_vec = np.load("./edge_attr_directed_v2.npy")
        edge_vec = torch.tensor(edge_vec, dtype=torch.float)
        data.x = torch.cat([data.x, edge_vec], dim=-1)
        print("Node features after add edge attr:", data.x.size())
    if use_edge_ts:

        edge_vec = np.load("./edge_ts_v2.npy")
        edge_vec = torch.tensor(edge_vec, dtype=torch.float)
        data.x = torch.cat([data.x, edge_vec], dim=-1)

        edge_vec = np.load("./edge_ts_v3.npy")
        edge_vec = torch.tensor(edge_vec, dtype=torch.float)
        data.x = torch.cat([data.x, edge_vec], dim=-1)
        print("Node features after add edge ts:", data.x.size())

    if use_back_nodes:
        edge_vec = np.load("./back_nodes_vec.npy")
        edge_vec = torch.tensor(edge_vec, dtype=torch.float)
        data.x = torch.cat([data.x, edge_vec], dim=-1)
        print("Node features after add edge ts:", data.x.size())

    # data.x = data.x[:,1:]

    # data.x = torch.cat([data.x, torch.tensor(np.load('./notebook/node2vec.npy'))], dim=-1)

    if data.y.dim()==2:
        data.y = data.y.squeeze(1)

    split_idx = {'train':data.train_mask, 'valid':data.valid_mask, 'test':data.test_mask}

    data = data.to(device)

    train_idx = split_idx['train']

    # # # # All nodes
    # tmp_set = set(list(data.train_mask.cpu().numpy())  + list(data.valid_mask.cpu().numpy()) + list(data.test_mask.cpu().numpy()))
    # bg_idx = []
    # for i in range(len(data.y)):
    #     if i not in tmp_set:
    #         bg_idx.append(i)

    # bg_idx = torch.tensor(bg_idx)
    # train_idx = torch.cat([train_idx, torch.tensor(bg_idx)]).to(device)

    model_dir = prepare_folder(args.dataset, args.model)
    print('model_dir:', model_dir)

    # , 15, 12, 10
    train_loader = NeighborSampler(data.adj_t, node_idx=train_idx, sizes=[10, 5, 5],
                                   batch_size=1024, shuffle=True, num_workers=12)

    # train_loader_nolabel = NeighborSampler(data.adj_t, node_idx=bg_idx, sizes=[20, 15],
    #                                batch_size=2048, shuffle=True, num_workers=12)

    # sampler = ImbalancedSampler(data, input_nodes=train_idx)
    # train_loader = NeighborLoader(data, input_nodes=train_idx, num_neighbors=[10, 7, 4], sampler=sampler, batch_size=2048, shuffle=True, num_workers=12)
    # test_idx = torch.cat([data.train_mask, data.valid_mask, data.test_mask])

    # layer_loader = NeighborSampler(data.adj_t, node_idx=None, sizes=[-1], batch_size=4096, shuffle=False, num_workers=12)

    # # All nodes
    valid_idx = split_idx['valid']
    # tmp_set = set(list(data.train_mask.cpu().numpy()) + list(data.test_mask.cpu().numpy()))
    # bg_idx = []
    # for i in range(len(data.y)):
    #     if i not in tmp_set:
    #         bg_idx.append(i)
    # valid_idx = torch.cat([valid_idx, torch.tensor(bg_idx)]).to(device)

    test_idx = split_idx['test']
    # tmp_set = set(list(data.train_mask.cpu().numpy()) + list(data.valid_mask.cpu().numpy()))
    # bg_idx = []
    # for i in range(len(data.y)):
    #     if i not in tmp_set:
    #         bg_idx.append(i)
    # test_idx = torch.cat([test_idx, torch.tensor(bg_idx)]).to(device)

    train_loader_v2 = NeighborSampler(data.adj_t, node_idx=train_idx, sizes=[-1, -1, -1], batch_size=4096, shuffle=False, num_workers=12)
    valid_loader = NeighborSampler(data.adj_t, node_idx=valid_idx, sizes=[-1, -1, -1], batch_size=4096, shuffle=False, num_workers=12)
    test_loader = NeighborSampler(data.adj_t, node_idx=test_idx, sizes=[-1, -1, -1], batch_size=4096, shuffle=False, num_workers=12)

    # train_loader_v2 = NeighborSampler(data.adj_t, node_idx=train_idx, sizes=[-1, -1, -1, -1, -1], batch_size=4096, shuffle=False, num_workers=12)
    # valid_loader = NeighborSampler(data.adj_t, node_idx=valid_idx, sizes=[-1, -1, -1, -1, -1], batch_size=4096, shuffle=False, num_workers=12)
    # test_loader = NeighborSampler(data.adj_t, node_idx=test_idx, sizes=[-1, -1, -1, -1, -1], batch_size=4096, shuffle=False, num_workers=12)

    if args.model == 'sage_neighsampler':
        para_dict = sage_neighsampler_parameters
        model_para = sage_neighsampler_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = SAGE_NeighSampler(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)
    if args.model == 'sage_neighsampler_ens':
        para_dict = sage_neighsampler_parameters
        model_para = sage_neighsampler_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = SAGE_NeighSamplerEnsemble(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)

    if args.model == 'gat_neighsampler':
        para_dict = gat_neighsampler_parameters
        model_para = gat_neighsampler_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = GAT_NeighSampler(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)
    if args.model == 'gatv2_neighsampler':
        para_dict = gatv2_neighsampler_parameters
        model_para = gatv2_neighsampler_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = GATv2_NeighSampler(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)

    if args.model == 'gcn2ensemble':
        para_dict = gcn2ensemble_parameters
        model_para = gcn2ensemble_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = GCN2ConvEnsemble(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)
    if args.model == 'gcn2':
        para_dict = gcn2_parameters
        model_para = gcn2_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = GCN2Net(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)

    if args.model == 'trans':
        para_dict = trans_parameters
        model_para = trans_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = TransNetNS(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)

    if args.model == 'transv2':
        para_dict = transv2_parameters
        model_para = transv2_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = TransNetNSV2(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)
    if args.model == 'transv3':
        para_dict = transv3_parameters
        model_para = transv3_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = TransNetNSV3(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)

    if args.model == 'pna':
        para_dict = pna_parameters
        model_para = pna_parameters.copy()
        model_para.pop('lr')
        model_para.pop('l2')
        model = PNANetNS(in_channels = data.x.size(-1), out_channels = nlabels, **model_para).to(device)

    print(f'Model {args.model} initialized')

    # raw_data = np.load('xydata/raw/phase1_gdata.npz')
    # ts = np.concatenate([raw_data['edge_timestamp'], raw_data['edge_timestamp']])
    # ts = torch.tensor(ts).to(device)

    model.reset_parameters()
    optimizer = torch.optim.Adam(model.parameters(), lr=para_dict['lr'], weight_decay=para_dict['l2'])
    # optimizer = torch.optim.SparseAdam(list(model.parameters()), lr=para_dict['lr'])

    # optimizer = torch.optim.SGD(model.parameters(), lr=para_dict['lr'])
    min_valid_loss = 1e8
    max_valid_auc = 1e-8

    for epoch in range(1, args.epochs+1):
        loss = train(epoch, train_loader, model, data, train_idx, optimizer, device, no_conv)

        # loss_2 = train_nolabel(epoch, train_loader_nolabel, model, data, bg_idx, optimizer, device, no_conv)

        # losses, out = test(layer_loader, model, data, split_idx, device, no_conv)
        losses, out = test_v2(train_loader_v2, valid_loader, test_loader, model, data, split_idx, device, no_conv)
        train_loss, valid_loss, test_loss = losses['train'], losses['valid'], losses['test']

        evaluator = Evaluator('auc')
        preds_train, preds_valid = out[data.train_mask], out[data.valid_mask]
        y_train, y_valid = data.y[data.train_mask], data.y[data.valid_mask]
        train_auc = evaluator.eval(y_train, preds_train)['auc']
        valid_auc = evaluator.eval(y_valid, preds_valid)['auc']
        print('train_auc:',train_auc)
        print('valid_auc:',valid_auc)

        if valid_loss < min_valid_loss:
            min_valid_loss = valid_loss
            torch.save(model.state_dict(), model_dir+'model.pt')

        if valid_auc > max_valid_auc:
            max_valid_auc = valid_auc
            torch.save(model.state_dict(), model_dir+'model.pt')

        if epoch % args.log_steps == 0:
            print(f'Epoch: {epoch:02d}, '
                      f'Loss: {loss:.4f}, '
                    #   f'Loss_NoLabel: {loss_2:.4f}, '
                      f'Train: {100 * train_loss:.3f}%, '
                      f'Valid: {100 * valid_loss:.3f}% '
                      f'Test: {100 * test_loss:.3f}%')

if __name__ == "__main__":
    main()


from django.conf.urls.defaults import *
from django.views.generic.simple import direct_to_template
from django.views.generic.list_detail import object_list
from django.contrib.auth.views import login, logout
from django.contrib import admin

from models import Chat
from views import chat, post, create, register, ping, filterchat, username_available

# These options dicts set up the front page (index) and public chat
# list views, which use Django's generic views and therefore do not
# have their own dedicated view functions in views.py.
index_options = {
    'template': 'index.html',
    'extra_context': {
        # Show the five most recently-updated chats on the front page
        'public_chats': lambda: Chat.objects.public()[:5],
    },
}
chat_list_options = {
    'queryset': Chat.objects.public().order_by('name'),
    'template_object_name': 'chat',
    'template_name': 'chat/chat_list.html',
}

# The URL patterns for Watercooler.  The names given to some of the
# patterns makes it easy to generate those URLs via the reverse()
# function in views or the {% url %} template tag in templates (to
# avoid hardcoding URLs).
urlpatterns = patterns('',
    # Essentially static pages
    url(r'^$', direct_to_template, index_options, name='index'),
    url(r'^about/$', direct_to_template, {'template': 'about.html'}, name='about'),
    url(r'^contact/$', direct_to_template, {'template': 'contact.html'}, name='contact'),

    # Create a new chat
    url(r'^create/$', create, name='create'),

    # The list of public chats
    url(r'^chat/$', object_list, chat_list_options, name='public_chats'),

    # Views for individual chats, identified by their slug
    url(r'^chat/(?P<slug>[-\w]+)/$', chat, name='chat'),
    url(r'^chat/(?P<slug>[-\w]+)/post/', post, name='post'),
    url(r'^chat/(?P<slug>[-\w]+)/ping/', ping, name='ping'),
    url(r'^chat/(?P<slug>[-\w]+)/filter/', filterchat, name='filter'),

    # Authentication views (login and logout are provided by the
    # Django auth framework, register is provided in views.py)
    (r'^accounts/login/$',  login),
    (r'^accounts/logout/$', logout),
    (r'^accounts/register/$', register),
    (r'^accounts/register/usernameAvailable/(?P<username>.*)/$', username_available),

    # The Django admin site
    (r'^admin/(.*)', admin.site.root),
)

# Automatically discover and install any admin sites (in this case,
# just the one admin.py)
admin.autodiscover()


# standard libraries
import cv2
import numpy as np
from typing import List, Dict
from PIL import ImageGrab, ImageTk, Image
from pytesseract import Output, pytesseract
from mvc.custom_dataclass import TextPosition

def filter_text_points(dict_of_words: Dict[str, str], list_of_text_and_position: List[TextPosition]) -> List[TextPosition]:
    word_positions = {text_and_position.text: text_and_position for text_and_position in list_of_text_and_position}
    func_result = []

    for key, word in dict_of_words.items():
        if key in word_positions:
            text_position = word_positions[key]
            text_position.text = word
            func_result.append(text_position)

    return func_result

def image_to_text_position(img):
    """"""
    res_list = []
    ocr_data = pytesseract.image_to_data(img, output_type=Output.DICT)
    for index, text in enumerate(ocr_data["text"]):
        if text != "":
            """"""
            text = text
            x1 = int(ocr_data["left"][index])
            y1 = int(ocr_data["top"][index])
            x2 = x1 + int(ocr_data["width"][index])
            y2 = y1 + int(ocr_data["height"][index])
            new_text_position_point = TextPosition(text, x1, y1, x2, y2)
            res_list.append(new_text_position_point)
    return res_list

def grab_screen_shot():
        try:
            # Get the screenshot from the clipboard
            image = ImageGrab.grabclipboard()
            # if image = NONE ( no value ) then if statement doesn't get executed
            if image:
                return image
            else:
                print("No screenshot available")
        except Exception as e:
            print(e)

def image_transform(image_path: str) -> np.ndarray:
    # Load the image
    img = cv2.imread(image_path)
    # Convert the image to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # Threshold the image
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
    # Remove noise
    thresh = cv2.medianBlur(thresh, 3)
    # Perform OCR on the image
    return img, gray

def draw_rects_on_img(img: np.ndarray, text_positions: TextPosition) -> Image:
    # font
    font = cv2.FONT_HERSHEY_SIMPLEX
    fontScale = 0.5
    color = (255, 0, 0)
    thickness = 1
    for tp_point in text_positions:
    # aliases for textbox position variables
        x1 = tp_point.x1
        y1 = tp_point.y1
        x2 = tp_point.x2
        y2 = tp_point.y2
        text_y_offset = 5
        # Draw rectangles and text on the given positions
        cv2.putText(img, tp_point.text, (x1,y1-text_y_offset), font, fontScale, color, thickness)
        # 2 is the offset, so the rectangle doesn't get drawn over the words
        cv2.rectangle(img, (x1-2, y1-2), (x2+2, y2+2), (0, 255, 0), 1)
        # return the completed image
    return img

def show_image(img:np.ndarray):
    """"""
    cv2.imshow('Originial', img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()


# SAFESYS 0 - Camera Management Module
# Capturing video from webcam w/ opencv
# Written by Kyle Christie xx/7/2020

#library imports
import numpy as np                 # needed for opencv to work
import cv2                         # opencv module for camera manipulation/usage
from datetime import datetime      # datetime used when naming files
import winsound                    # winsound to allow beeping
import os                          # os module used for file management

cap = cv2.VideoCapture(0)

# Capture frame-by-frame
ret, frame = cap.read()
font = cv2.FONT_HERSHEY_SIMPLEX

#write text and place a red circle on the camera GUI
gray = cv2.putText(frame ,'You have been caught!',(10,100), font, 1,(0,0,0),2,cv2.LINE_AA)
cv2.circle(frame,(575,75), 63, (0,0,255), -1)

# try to create folder - handle error (implemented to clean up working directory/have a dedicated area for safesys alerts)
try:
    os.mkdir("Safesys Captures")
except:
    print("Folder already exists")

#use date/time to generate part of the filename
now = datetime.now()
current_time = now.strftime("%H_%M_%S")

winsound.Beep(500,200)

#concat date/time into the below text to create the filename
file_name = "Safesys Captures/safesys_alert_" + current_time + ".png"
print("Saving... " + file_name)

winsound.Beep(300,200)

#save the image to current dir
cv2.imwrite(file_name, frame)

winsound.Beep(200,200)

#Show the UI & img to the user
cv2.imshow('frame', gray)

# When everythings done, release the capture
cap.release()
cv2.destroyAllWindows()


"""Extraction de segments de textes avec CRF"""

import csv
import itertools
import re
from pathlib import Path
from typing import Any, Iterable, Iterator, Sequence

import joblib  # type: ignore

from alexi.analyse import group_iob
from alexi.convert import FIELDNAMES, Converteur
from alexi.segment import Segmenteur, T_obj, page2features, page2labels, split_pages

FEATNAMES = [name for name in FIELDNAMES if name != "sequence"]
DEFAULT_MODEL = Path(__file__).parent / "models" / "crfseq.joblib.gz"
NUMDASH = re.compile(r"[\d-]+")

def features(page: Sequence[T_obj]) -> Iterator[list[str]]:
    for word in page:
        features = ["bias"]
        features.append("lower=%s" % word["text"].lower())
        features.append("reg=%s" % bool(word["text"].lower() == "règlement"))
        features.append("n=%s" % bool(word["text"][0] in "Nn"))
        features.append("alpha=%s" % bool(word["text"].isalpha()))
        features.append("numdash=%s" % bool(NUMDASH.match(word["text"])))
        features.append("bold=%s" % bool("bold" in word["fontname"].lower()))
        features.append("tag=%s" % word["segment"].partition("-")[2])
        features.append("size=%d" % (int(word["bottom"]) - int(word["top"])))
        yield features

def labels(_, word):
    return word.get("sequence", "O")

def load(paths):
    for p in paths:
        with open(Path(p), "rt") as infh:
            reader = csv.DictReader(infh)
            last_page = None
            for page in split_pages(reader):
                if last_page is None:
                    yield page
                last_page = page
            yield last_page

def make_data(dataset, n=2):
    return zip(
        *((page2features(s, features, n), page2labels(s, labels)) for s in dataset)
    )

class Extracteur:
    def __init__(self, model=DEFAULT_MODEL):
        self.crf = joblib.load(model)

    def predict(self, words: Iterable[dict[str, Any]]) -> Iterable[dict[str, Any]]:
        first_page = last_page = None
        for page in split_pages(words):
            if last_page is None:
                first_page = page
            else:
                if last_page is first_page:
                    yield from self.crf.predict_single(
                        page2features(last_page, feature_func=features, n=2)
                    )
                else:
                    for _ in last_page:
                        yield "O"
            last_page = page
        if last_page:
            yield from self.crf.predict_single(
                page2features(last_page, feature_func=features, n=2)
            )

    def __call__(self, words: Iterable[dict[str, Any]]) -> Iterable[dict[str, Any]]:
        c1, c2 = itertools.tee(words)
        pred = self.predict(c1)
        for label, word in zip(pred, c2):
            word["sequence"] = label
            yield word

def main():
    import argparse

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("pdfpath", type=Path, help="Fichier PDF")
    args = parser.parse_args()
    conv = Converteur(args.pdfpath)
    seg = Segmenteur()
    ex = Extracteur()
    pages = conv.extract_words()
    segmented = seg(pages)
    tagged = ex(segmented)
    for bloc in group_iob(tagged, "sequence"):
        print(f"{bloc.type}: {bloc.texte}")

if __name__ == "__main__":
    main()


import requests
from requests.exceptions import HTTPError
import re
from bs4 import BeautifulSoup
import itertools
import time

# See https://stackoverflow.com/a/8991553/51685
def chunker(n, iterable):
    it = iter(iterable)
    while True:
        chunk = tuple(itertools.islice(it, n))
        if not chunk:
            return
        yield chunk

def get_ultra_miner_floor() -> list:
    url_headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
    }
    buy_now_url = "https://opensea.io/collection/ultra-miners?search[sortAscending]=true&search[sortBy]=PRICE&search[toggles][0]=BUY_NOW"
    status_code = 999
    while status_code != 200:
        try:
            response = requests.get(url=buy_now_url, headers=url_headers)
            status_code = response.status_code
            # If the response was successful, no Exception will be raised
            response.raise_for_status()
        except HTTPError as http_err:
            print(f"HTTP error occurred: {http_err}")
            print("sleep for 1 second")
            time.sleep(1)
        except Exception as err:
            print(f"Other error occurred: {err}")
            break

    html = str(response.content)

    """
    Use regex to match ULTRA MINER #XXXX
    """
    regex = r"ULTRA MINER \#[0-9]{2,5}"
    pattern = re.compile(regex)
    matches_list = re.findall(pattern, html)
    # print(f"matches_list: {matches_list}")
    """
    Strip "ULTRA MINER #XXXX" to Int
    """
    nft_token_id = list()
    for id in matches_list:
        int_val = int(id.strip("ULTRA MINER #"))
        nft_token_id.append(int_val)
    # print(f"nft_token_id:{nft_token_id}")
    """Remove duplicates"""
    nft_token_id = list(set(nft_token_id))
    return nft_token_id

def get_ultra_price(miner_nft_id: str) -> str:
    BASE_URL = "https://opensea.io/assets"
    NFT_CONTRACT_ADDR = "0x0c6822ca73de6871f27acd9ca05a05b99294b805"

    custom_headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
    }

    REQ_URL = BASE_URL + "/" + NFT_CONTRACT_ADDR + "/" + str(miner_nft_id)
    status_code = 999
    while status_code != 200:
        try:
            response = requests.get(REQ_URL, headers=custom_headers)
            status_code = response.status_code
            # If the response was successful, no Exception will be raised
            response.raise_for_status()
        except HTTPError as http_err:
            print(f"HTTP error occurred: {http_err}")
            print("sleep for 1 second")
            time.sleep(1)
        except Exception as err:
            print(f"Other error occurred: {err}")
            break
    soup = BeautifulSoup(response.text, "html.parser")
    soup_val = soup.find(class_="Price--amount")

    return soup_val.get_text().strip() if soup_val else ""


# -*- coding: UTF-8 -*-
"""
    梯度下降求解y=x**2的最小值
"""
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt

from mpl_toolkits.mplot3d import Axes3D

# 设置字符集，防止中文乱码
mpl.rcParams['font.sans-serif'] = [u'simHei']
mpl.rcParams['axes.unicode_minus'] = False

# 原函数
def f(x, y):
    return x**2 + y**2

# 偏导
def h(t):
    return 2 * t

# 画图
def draw(X, Y, Z, x, y, f_current):
    figure = plt.figure()
    axes = Axes3D(figure)
    X2 = np.arange(-2, 2, 0.2)
    Y2 = np.arange(-2, 2, 0.2)
    X2, Y2 = np.meshgrid(X2, Y2)
    Z2 = f(X2, Y2)

    axes.plot_surface(X2, Y2, Z2, rstride=1, cstride=1, cmap='rainbow')
    axes.plot(X, Y, Z, 'ro--')
    axes.set_title(u'z=x^2+y^2函数求解最小值，最终解为：x=%.2f,y=%.2f,z=%.2f'
        % (x, y, f_current))
    plt.show()

# 梯度下降求解z=x**2+y**2的最小值
def gd():
    X = []
    Y = []
    Z = []

    x = 2
    y = 2
    step = 0.1
    f_change = f(x, y)
    f_current = f(x, y)
    X.append(x)
    Y.append(y)
    Z.append(f_current)
    while f_change > 1e-10:
        x = x - step * h(x)
        y = y - step * h(y)
        f_change = np.abs(f_current - f(x, y))
        f_current = f(x, y)

        X.append(x)
        Y.append(y)
        Z.append(f_current)
    print("result: ", (x, y, f_current))
    draw(X, Y, Z, x, y, f_current)

if __name__ == '__main__':
    gd()


"""Add users.created_at

Revision ID: 9d9d679be956
Revises: 6155a82fa0cc
Create Date: 2023-12-14 01:35:23.863962

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '9d9d679be956'
down_revision = '6155a82fa0cc'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('users', schema=None) as batch_op:
        batch_op.add_column(sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False))

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('users', schema=None) as batch_op:
        batch_op.drop_column('created_at')

    # ### end Alembic commands ###


""" Library of functions to build the CCD calibration files for image processing to reduce data from the Tull coude spectrograph

Created by DMK on 9/24/2023

Last updated by DMK on 10/14/2023
"""

##### Imports

import numpy as np
import matplotlib.pyplot as plt

from astropy.io import fits
from os import path

##### Functions

def build_super_bias( bias_file_names, read_noise ):
    """ Function to read in the bias images and median combine them into a super bias to apply to other frames

    Parameters
    ----------
    bias_file_names : list
        List of file names (type str) for the raw bias frames
    read_noise : float
        The read noise (in ADU, converted from the header value which is given in electrons)

    Returns
    -------
    super_bias : HDUList
        Multi-extension FITS ImageHDU list with the median combined super bias and associated error
    """

    # Initialize an array to hold the bias images
    bias_cube = []

    for i_file, file_name in enumerate( bias_file_names ):

        # Read in the bias file
        bias_image = fits.getdata( file_name )

        # Append to the cube of all bias images
        bias_cube.append( bias_image )

    # Get the super bias -- median combine the bias images, and add in read noise to bias error
    super_bias_flux  = np.nanmedian( bias_cube, axis = 0 )
    super_bias_error = np.sqrt( super_bias_flux + read_noise ** 2.0 )

    ### Output -- into FITS object

    # Put the super bias data into FITS ImageHDU objects
    super_bias_flux_hdu  = fits.ImageHDU( super_bias_flux, name = 'bias flux' )
    super_bias_error_hdu = fits.ImageHDU( super_bias_error, name = 'bias error' )

    # Output to a FITS multi-extension HDU object (with an empty Primary HDU to hold some header information)
    super_bias = fits.HDUList( [ fits.PrimaryHDU(), super_bias_flux_hdu, super_bias_error_hdu ] )

    # Add some keywords to the output super bias object
    super_bias[0].header['nbiases'] = ( len( bias_file_names ), 'Number of bias frames combined' )
    super_bias[0].header['rdnoise'] = ( read_noise, 'Read noise added to error' )
    super_bias[0].header['history'] = 'Median combined super bias frame'

    return super_bias

def build_flat_field( flat_file_names, read_noise, super_bias ):
    """ Function to read in flat field images and median combine them into the flat field for image processing

    Parameters
    ----------
    flat_file_names : list
        List of file names (type str) for the raw flat field frames
    read_noise : float
        The read noise (in ADU, converted from the header value which is given in electrons)
    super_bias : HDUList
        Multi-extension FITS ImageHDU list with the median combined super bias and associated error created with build_super_bias.

    Returns
    -------
    flat_field : HDUList
        Multi-extension FITS ImageHDU list with the median combined flat field and associated error.
    """

    # Initialize an array to hold the flat field images
    flat_cube = []

    for i_file, file_name in enumerate( flat_file_names ):

        # Read in the bias file
        flat_image = fits.getdata( file_name )

        # Append to the cube of all bias images
        flat_cube.append( flat_image )

    # Get the flat field -- median combine the flat field images and subtract the super bias
    flat_field_median = np.nanmedian( flat_cube, axis = 0 )
    flat_field_flux   = flat_field_median - super_bias['bias flux'].data

    # Get the flat field errors (flat field, read noise, and super bias)
    flat_field_error = np.sqrt( flat_field_median + read_noise ** 2.0 + super_bias['bias error'].data ** 2.0 )

    ### Output

    # Put the flat field flux and error into ImageHDU objects, with minor math to make the flat field scale between 0 and 1
    flat_field_flux_hdu  = fits.ImageHDU( ( flat_field_flux - np.nanmin( flat_field_flux ) ) / np.nanmax( flat_field_flux - np.nanmin( flat_field_flux ) ), name = 'flat flux' )
    flat_field_error_hdu = fits.ImageHDU( flat_field_error / np.nanmax( flat_field_flux - np.nanmin( flat_field_flux ) ), name = 'flat error' )

    # Put into an HDUList
    flat_field = fits.HDUList( [ fits.PrimaryHDU(), flat_field_flux_hdu, flat_field_error_hdu ] )

    # Add some keywords to the output super bias object
    flat_field[0].header['nflats']  = ( len( flat_file_names ), 'Number of flat field frames combined' )
    flat_field[0].header['rdnoise'] = ( read_noise, 'Read noise added to error' )
    flat_field[0].header['history'] = 'Median combined flat field frame'
    flat_field[0].header['history'] = 'Super bias subtracted'

    return flat_field

def make_bad_pixel_mask( super_bias, flat_field, bias_bpm_percentile, flat_field_bpm_limit ):
    """ Function to create a bad pixel mask using the super bias and flat field.

    Parameters
    ----------
    super_bias : HDUList
        Multi-extension FITS ImageHDU list with the median combined super bias and associated error created with build_super_bias.
    flat_field : HDUList
        Multi-extension FITS ImageHDU list with the median combined flat field and associated error created with build_flat_field.
    bias_bpm_percentile : float
        The percentile above which super bias counts are marked as bad pixels (hot pixels). This is defined in the config file.
    flat_field_bpm_limit : float
        The flat field value below which a pixel is marked as bad. This is defined in the config file.

    Returns
    -------
    bad_pixel_mask : FITSHDU
        The bad pixel mask created using the super bias and flat field. Bad pixels are valued 0, good pixels are valued 1.
    """

    # Get the super bias flux value for the percentile above which to label bad pixels, as defined in the top level config
    bias_percentile_cut_value = np.nanpercentile( super_bias['bias flux'].data, bias_bpm_percentile )

    # Define the bad pixel mask as indices where the super bias flux is above the percentile limit, or the flat field is very small
    bad_pixel_mask_indices = np.where( ( super_bias['bias flux'].data > bias_percentile_cut_value ) | ( flat_field['flat flux'].data < flat_field_bpm_limit ) )

    ### Output the bad pixel mask as a fits file: 0s are bad pixels, 1s are good pixels

    bad_pixel_mask = np.ones( super_bias['bias flux'].data.shape )

    bad_pixel_mask[bad_pixel_mask_indices] = 0

    # FITS HDU
    bad_pixel_mask_hdu = fits.PrimaryHDU( bad_pixel_mask )

    bad_pixel_mask_hdu.header['history'] = 'Bias above {:.2f}'.format( bias_percentile_cut_value )
    bad_pixel_mask_hdu.header['history'] = 'Flat field below {:.5f}'.format( flat_field_bpm_limit )

    return bad_pixel_mask_hdu

def cal_image_2d_plot( image, figure_title, file_name, bpm = None ):
    """ Function to make a 2D image plot of an input 2D data array. Generalized to plot any input image.

    Parameters
    ----------
    image : array
        The 2D array to show.
    title : str
        The title of the plot (here to mark what type of image is plotted).
    file_name : str
        The file name to save the plot to.
    bpm : array, optional
        The 2D array defining the bad pixel mask. This can be input and overplotted the input data image. The default is None.

    Returns
    -------
    None.
    """

    plt.clf()

    # Plot the image
    plt.imshow( np.log10( image ), cmap = 'gray', aspect = 'auto', interpolation = 'none' )

    # If the bad pixel mask is given, plot the bad pixels on top!
    if bpm is not None:
        bpm_pixels = np.where( bpm == 0 )

        # Invert the x and y BPM for imshow
        plt.plot( bpm_pixels[1], bpm_pixels[0], '.', ms = 1, c = '#db6d1b', label = 'Bad pixel mask' )

        plt.legend( fontsize = 'small' )

    # Add colorbar
    plt.colorbar()

    # Add title passed to the function
    plt.title( figure_title )

    # Save the figure
    plt.savefig( file_name, bbox_inches = 'tight', pad_inches = 0.05 )
    plt.clf()

    return None

##### Wrapper for building all calibration files

def build_calibrations( header_df, bias_frame_indices, flat_frame_indices, config ):
    """ Main function to build all of the CCD calibrations, using the above functions for the individual calibrations.

    Parameters
    ----------
    header_df : pandas DataFrame
        The compiled information from the file headers.
    bias_frame_indices : list
        List of indices of the bias frames for the header dataframe.
    flat_frame_indices : list
        List of indices of the flat frames for the header dataframe.
    config : dict
        The overall config file defined using YAML with all parameters for running the reduction and analysis pipeline.

    Returns
    -------
    None.
    """

    ### Build the files

    # First the super bias
    print( 'Reading bias files and creating super bias' )

    super_bias = build_super_bias( header_df['file_name'].values[bias_frame_indices], ( header_df['read_noise'] / header_df['gain'] ).values[bias_frame_indices][0] )
    super_bias.writeto( path.join( config['general']['reduction_dir'], 'cals', 'super_bias.fits' ), overwrite = True, checksum = True )

    # Next the flat field
    print( 'Reading flat files and creating flat field' )

    flat_field = build_flat_field( header_df['file_name'].values[flat_frame_indices], ( header_df['read_noise'] / header_df['gain'] ).values[flat_frame_indices][0], super_bias )
    flat_field.writeto( path.join( config['general']['reduction_dir'], 'cals', 'flat_field.fits' ), overwrite = True, checksum = True )

    # Last the bad pixel mask
    print( 'Building the bad pixel mask from the super bias and flat field' )

    bad_pixel_mask = make_bad_pixel_mask( super_bias, flat_field, config['calibrations']['bias_bpm_percentile'], config['calibrations']['flat_field_bpm_limit'] )
    bad_pixel_mask.writeto( path.join( config['general']['reduction_dir'], 'cals', 'bad_pixel_mask.fits' ), overwrite = True, checksum = True )

    ### Make quick check plots for the calibration files!

    # First for the super bias
    super_bias_title = 'Super Bias, Median Value = {0:.2f}'.format( np.nanmedian( super_bias['bias flux'].data ) )
    cal_image_2d_plot( super_bias['bias flux'].data, super_bias_title, path.join( config['general']['reduction_dir'], 'cals', 'super_bias.pdf' ) )

    # Next for the flat fiel
    flat_field_title = 'Flat Field'
    cal_image_2d_plot( flat_field['flat flux'].data, flat_field_title, path.join( config['general']['reduction_dir'], 'cals', 'flat_field.pdf' ) )

    # Last for the bad pixel mask
    bpm_title = 'Bad Pixel Mask, plotted on Super Bias'
    cal_image_2d_plot( super_bias['bias flux'].data, bpm_title, path.join( config['general']['reduction_dir'], 'cals', 'bad_pixel_mask.pdf' ), bpm = bad_pixel_mask.data )

    return None


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 12 08:08:32 2020

@author: z011348
"""
import numpy as np
import pandas as pd
heart_disease = pd.read_csv('C:/Users/z011348/Desktop/ML/input/heart-disease.csv')

# setup random see
np.random.seed(42)

# make the data
X = heart_disease.drop("target", axis=1)
y = heart_disease["target"]

# split data into training and test data
from sklearn.model_selection import train_test_split

# =============================================
# lets try the RandomForest classifier
# =============================================
from sklearn.ensemble import RandomForestClassifier

# split data into training and test data
X_train, X_test, y_train, y_test = train_test_split(X,
                                                    y,
                                                    test_size=0.2)
# Instantiate RandomForestClassifier()
rc = RandomForestClassifier()
rc.fit(X_train, y_train)

# Check/evaluate the model
print(rc.score(X_test, y_test))     # 0.8524590163934426

# Use trained model to make the predictions - predict()
y_pred = rc.predict(X_test)
print(y_pred[:5])

# Compare prdictions to truth lables to evaluate the model - one way
print(np.mean(y_pred == y_test))

# Compare prdictions to truth lables to evaluate the model - another way
from sklearn.metrics import accuracy_score
print(accuracy_score(y_test, y_pred))

# Use trained model to make the predictions - predict_proba()
# predict_proba() returns probabilities of a classification label
y_pred_proba = rc.predict_proba(X_test[:5])
print(y_pred_proba)


import json
import sys

import numpy as np

def main():
    data = json.load(sys.stdin)
    ev_curves = data["ev_curves"]

    v_0_data = []
    b_0_data = []

    for ev_curve in ev_curves:
        fit = ev_curve["eos_fit"]

        if fit["eos"] != "birch_murnaghan":
            continue

        params = fit["params"]

        v_0_data.append(params["V0"])
        b_0_data.append(params["B0"])

    v_0_data = np.array(v_0_data)
    b_0_data = np.array(b_0_data)

    print("V_0: {:.2f} +/- {:.2f}".format(v_0_data.mean(), v_0_data.std()))
    print("B_0: {:.2f} +/- {:.2f}".format(b_0_data.mean(), b_0_data.std()))

if __name__ == "__main__":
    main()


import numpy as np
import cv2

im = cv2.imread('phanvunganh/balloons.jpg')
blur=cv2.GaussianBlur(im,(3,3),0)       #Làm mờ hình ảnh bằng bộ lọc Gaussian.

imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)       #Biến ảnh màu thành ảnh xám
cv2.imshow("imgray", imgray)

res, thresh = cv2.threshold(src=imgray, thresh=230, maxval=255, type=cv2.THRESH_BINARY)      #Chuyển ảnh xám thành ảnh nhị phân
#res, thresh = cv2.threshold(imgray, 0, 25, 0)
#res, thresh = cv2.threshold(imgray, 0, 100, 0)
#res, thresh = cv2.threshold(src=imgray,thresh=127,maxval=255,type=cv2.THRESH_BINARY)
#thresh = cv2.Canny(imgray, 60, 150)                     # nhị phân hóa ảnh

        #src=tham số đầu tiên là 1 ảnh xám,
        #thresh=tham số thứ 2 là giá trị ngưỡng,
        #maxval=tham số thứ 3 maxval là giá trị được gán nếu giá pixel lớn hơn giá trị ngưỡng,
        #type=tham số thứ 4 là loại phân ngưỡng

cv2.imshow("thresh", thresh)

#contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)       #Tìm đường viền trong ảnh nhị phân
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)

        #image : hình ảnh cần tìm biên, là ảnh nhị phân.
        #contours : lưu trữ các đường biên tìm được, mỗi đường biên được lưu trữ dưới dạng một vector của các điểm.
        #hierarchy :  chứa thông tin về hình ảnh như số đường viền, xếp hạng các đường viền theo kích thước, trong ngoài, ..

print("contours size: ", str(len(contours)))
img = cv2.drawContours(im, contours, -1, (0,255,0), 3)       #Vẽ đường bao màu xanh dương

#img = cv2.drawContours(im, contours, 1, (0,255,0), 3)
#img = cv2.drawContours(im, contours, 3, (255, 0, 0), 3)

cv2.namedWindow("contour.jpg", 0)        #Mở 1 cửa sổ đặt tên là "contour.jpg"
cv2.imshow("contour.jpg", img)
cv2.waitKey(0)


from helpers import random_walk as rw
from random import random, choices, gauss
from math import sin, cos, radians
from collections import deque

from model.behavior import State
import numpy as np

from helpers.utils import get_orientation_from_vector, rotate

class AgentAPI:
    def __init__(self, agent):
        self.speed = agent.speed
        self.radius = agent.radius
        self.get_levy_turn_angle = agent.get_levy_turn_angle
        self.get_mu = agent.noise_mu

class Agent:
    colors = {State.EXPLORING: "black"}

    def __init__(self, robot_id, x, y, speed, radius,
                 bool_noise, noise_mu, noise_musd, noise_sd,
                 behavior, environment):

        self.id = robot_id
        self.pos = np.array([x, y]).astype('float64')
        self._speed = speed
        self._radius = radius
        self.orientation = random() * 360  # 360 degree angle

        self.bool_noise = bool_noise
        self.noise_mu = gauss(noise_mu, noise_musd)
        if random() >= 0.5:
            self.noise_mu = -self.noise_mu
        self.noise_sd = noise_sd

        self.environment = environment

        self.levy_counter = 1
        self.trace = deque(self.pos, maxlen=100)

        self.behavior = behavior
        self.api = AgentAPI(self)

    def __str__(self):
        return f"ID: {self.id}\n" \
               f"drift: {round(self.noise_mu, 5)}\n" \
               f"position: {np.round(self.pos, 2)}\n" \
               f"direction: {np.round(self.behavior.get_dr(), 2)}"

    def __repr__(self):
        return f"bot {self.id}"

    def step(self):
        sensors = self.environment.get_sensors(self)
        # self.check_food(sensors)
        self.behavior.step(sensors, AgentAPI(self))
        self.move()
        self.update_trace()

    def update_trace(self):
        self.trace.appendleft(self.pos[1])
        self.trace.appendleft(self.pos[0])

    def move(self):
        wanted_movement = rotate(self.behavior.get_dr(), self.orientation)
        if self.bool_noise:
            noise_angle = gauss(self.noise_mu, self.noise_sd)
            noisy_movement = rotate(wanted_movement, noise_angle)
            self.orientation = get_orientation_from_vector(noisy_movement)
            self.pos = self.clamp_to_map(self.pos + noisy_movement)
        else:
            self.orientation = get_orientation_from_vector(wanted_movement)
            self.pos = self.clamp_to_map(self.pos + wanted_movement)

    def clamp_to_map(self, new_position):
        if new_position[0] < self._radius:
            new_position[0] = self._radius
        if new_position[1] < self._radius:
            new_position[1] = self._radius
        if new_position[0] > self.environment.width - self._radius:
            new_position[0] = self.environment.width - self._radius
        if new_position[1] > self.environment.height - self._radius:
            new_position[1] = self.environment.height - self._radius
        return new_position

    def update_levy_counter(self):
        self.levy_counter -= 1
        if self.levy_counter <= 0:
            self.levy_counter = choices(range(1, rw.get_max_levy_steps() + 1), rw.get_levy_weights())[0]

    def get_levy_turn_angle(self):
        angle = 0
        if self.levy_counter <= 1:
            angle = choices(np.arange(0, 360), rw.get_crw_weights())[0]
        self.update_levy_counter()
        return angle

    def draw(self, canvas):
        circle = canvas.create_oval(self.pos[0] - self._radius,
                                    self.pos[1] - self._radius,
                                    self.pos[0] + self._radius,
                                    self.pos[1] + self._radius,
                                    fill=self.behavior.color,
                                    outline=self.colors[self.behavior.state],
                                    width=3)
        self.draw_comm_radius(canvas)
        self.draw_orientation(canvas)
        self.draw_trace(canvas)

    def draw_trace(self, canvas):
        tail = canvas.create_line(*self.trace)

    def draw_comm_radius(self, canvas):
        circle = canvas.create_oval(self.pos[0] - self.environment.robot_communication_radius,
                                    self.pos[1] - self.environment.robot_communication_radius,
                                    self.pos[0] + self.environment.robot_communication_radius,
                                    self.pos[1] + self.environment.robot_communication_radius,
                                    outline="gray")

    def draw_orientation(self, canvas):
        line = canvas.create_line(self.pos[0],
                                  self.pos[1],
                                  self.pos[0] + self._radius * cos(radians(self.orientation)),
                                  self.pos[1] + self._radius * sin(radians(self.orientation)),
                                  fill="white")

    def speed(self):
        return self._speed

    def radius(self):
        return self._radius


from argparse import ArgumentParser
import pandas as pd
import sys

class Travel:
    '''This displays a list of

    travel destinations.

    '''

    def __init__(self, filepath):

        '''Initializes Travel object'''

        self.filepath = pd.read_csv('travel_dest.csv')

    def travel_dataframe(self):    #Casslyn's Function

        '''This will display a dataframe of

        travel destinations ranked from 1-5.

        SideEffects: alters list of dataframe

        Returns: a sorted coloumn of ranked countries 1-5

        '''

        self_run = self.filepath.sort_values(by='Country', ascending=False)

        print(self_run)

    def travel(self):    #Casslyn's Function

        '''Returns a sorted coloumn of ranked countries 1-5 with

        a description.

        Args:

        best_rank: a variable that merges three dataframe columns together

        SideEffects: alters list of dataframe

        Returns: a list of ranked countries with their description

        '''

        best_rank = self.filepath[["Rank","Country","Don't miss"]]

        return best_rank

    def filter_distance(self,miles):    #Malik's Function

        '''Takes the travel dataframe and specified miles and

        returns a dataframe where specified values are true

        and the distance to the country is less than the miles specified.

        Args:

        dataframe(dataframe): resulting dataframe of travel

        miles(float): distance specified by the user

        Returns:

        results(dataframe): dataframe where all the desired values are true

        '''
        #for countries in dataframe:

        results = self.filepath[self.filepath["Distance"]<= miles].groupby("Country")["Rank"].min()
        #return results
        print(results)

    def most_popular(self):   #Malik's Function
        '''Find the max number of travelled for any location

        and return a list of the countries with number of people travelled.

        Args:

        dataframe (dataframe): resulting dataframe of travel

        Returns:

        popular(list): a list of countries with

        the max number of people travelled in the dataframe

        '''

        most_pop = self.filepath.max()

        most_travelled = self.filepath[self.filepath == most_pop].index.tolist()

        print(f'This is the most popular destinations traveled {most_travelled}')

    def user_input(self): #Casslyn's Fuction

        '''This asks for user input to get a list of

        hotels or a rank of family friendly countries

        Args:

        user_input: a string asking for user input

        list_x: a list containing keywords for user input

        SideEffects: Alters rank of list

        Retruns: a list of family friendly countries from least to greatest'''

        stay_dict = {"Nay Pald Hideaway":1,"Greenland Fairy Meadow Resort":2,

        "Velaa Private Island Resort":3,"wild camping in Mull":4,

        "Hotel Melia":5}

        key_stay = "Key box for countries: 1 = Philippines,2 = Pakistan,3 = Maldives,4 = Scotland,5 = Dubai"

        country_family_rank = {"Dubai":1,

        "Scotland":4,"Maldives":2,

        "Pakistan":3,"Philippines":5}

        n_2 = sorted(country_family_rank)

        while True:

            user_type = input('Type 1 for “where to stay” or 2 for “family friendly” to see a list of countries:' )

            if user_type == '1':

                print(key_stay,stay_dict)

                break

            if user_type == '2':

                print(n_2)

                break

def parse_args(arglist):

    '''Parse command-line arguments.

    Args:

    arglist (list of str): a list of command-line arguments.

    Returns:

    args: the parsed command-line arguments as a namespace

    '''

    parser = ArgumentParser()

    parser.add_argument("filepath", help="CSV containing travel destinations")
    parser.add_argument("miles", type=int, help= "Distance from countries")

    args = parser.parse_args(arglist)

    return args

def main(argmain,miles):

    '''Main will test and run all methods within Travel class.'''

    var = Travel(argmain)

    var.travel_dataframe()

    var.travel()

    var.filter_distance(miles)

    var.most_popular()

    var.user_input()

if __name__ == "__main__":

    '''This will invoke

    main() function.

    Returns: altered dataframes related to its method

    '''
    args = parse_args(sys.argv[1:])

    main(args.filepath,args.miles)


import pygame
from dronekit import connect,VehicleMode, LocationGlobalRelative,Vehicle
import time
from pymavlink import mavutil

pygame.init()
# sets the window title
pygame.display.set_caption(u'Keyboard events')
# sets the window size
s = pygame.display.set_mode((400, 400))

vehicle=connect("udp:127.0.0.1:14550",wait_ready=True)

def arm_and_takeoff(altitude):
    while not vehicle.is_armable:
        print("waiting to initialize.....")
        time.sleep(1)
    print("arming motors")

    vehicle.mode = VehicleMode("GUIDED")
    vehicle.armed = True

    while not vehicle.armed:
        print("waiting to arm")
        time.sleep(1)

    print("Taking off....")
    vehicle.simple_takeoff(altitude)

    while True:
        print('Altitude {}'.format(vehicle.location.global_relative_frame.alt))
        if vehicle.location.global_relative_frame.alt >= 0.95 * altitude:
            print("Target altitude reached ")
            break
        time.sleep(1)

def send_ned_velocity(Vx, Vy, Vz, duration):

    msg = vehicle.message_factory.set_position_target_local_ned_encode(
        0,
        0, 0,
        mavutil.mavlink.MAV_FRAME_LOCAL_NED,
        0b0000111111000111,
        0, 0, 0,
        Vx, Vy, Vz,
        0, 0, 0,
        0, 0)

    # send command to vehicle on 1 Hz cycle
    for i in range(0,duration):
        vehicle.send_mavlink(msg)

def condition_yaw(duration, heading, relative=False):
    if relative:
        is_relative=1 #yaw relative to direction of travel
    else:
        is_relative=0

    msg = vehicle.message_factory.command_long_encode(
        0, 0,
        mavutil.mavlink.MAV_CMD_CONDITION_YAW,
        0, #confirmation
        heading,                   # param 1, yaw in degrees
        0,                         # param 2, yaw speed deg/s
        1,                         # param 3, direction -1 ccw, 1 cw
        is_relative,               # param 4, relative offset 1, absolute angle 0
        0, 0, 0)                   # param 5 ~ 7 not used

    for i in range(0,duration):
        # send command to vehicle
        vehicle.send_mavlink(msg)

def Key():
    while True :
        event = pygame.event.wait()
        if event.type == pygame.QUIT:
            print("Returning to Launch")
            vehicle.mode = VehicleMode("RTL")

            break
        #--> pressed
        if event.type in (pygame.KEYDOWN, pygame.KEYUP):
            key_name = pygame.key.name(event.key)
            key_name = key_name.upper()
            if event.type == pygame.KEYDOWN:
            # prints on the console the key pressed
                print(u'"{}" key pressed'.format(key_name))
                if key_name == "W":
                    print("Move forward")
                    send_ned_velocity(10,0,0,5)
                    time.sleep(5)
                elif key_name == "A":
                    print("Move Left")
                    send_ned_velocity(0,0,0,0)
                elif key_name == "D":
                    print("Move Right")
                    send_ned_velocity(0,0,0,0)
                elif key_name == "S":
                    print("Move Back")
                    send_ned_velocity(-10,0,0,5)
                else:
                    print("Please Enter only W, A, S, D keys to control the vehicle")

arm_and_takeoff(30)
Key()