# Original source code developed by @clviper
# https://github.com/clviper/droidstatx

import gzip
import hashlib
import logging
import os
import re
import time
import xml.etree.ElementTree as ET
from io import BytesIO

from elftools.elf.elffile import ELFFile

import droidsf.utils
from droidsf.smali_checks import SmaliChecks

log = logging.getLogger(__name__)

NS_ANDROID_URI = "http://schemas.android.com/apk/res/android"
NS_ANDROID = "{http://schemas.android.com/apk/res/android}"

ANDROID_CODENAMES = {
    "3": "Cupcake 1.5",
    "4": "Donut 1.6",
    "5": "Eclair 2.0",
    "6": "Eclair 2.0.1",
    "7": "Eclair 2.1",
    "8": "Froyo 2.2.x",
    "9": "Gingerbread 2.3 - 2.3.2",
    "10": "Gingerbread 2.3.3 - 2.3.7",
    "11": "Honeycomb 3.0",
    "12": "Honeycomb 3.1",
    "13": "Honeycomb 3.2.x",
    "14": "Ice Cream Sandswich 4.0.1 - 4.0.2",
    "15": "Ice Cream Sandswich 4.0.3 - 4.0.4",
    "16": "Jelly Bean 4.1.x",
    "17": "Jelly Bean 4.2.x",
    "18": "Jelly Bean 4.3.x",
    "19": "KitKat 4.4 - 4.4.4",
    "21": "Lolipop 5.0",
    "22": "Lolipop 5.1",
    "23": "Marshmallow 6.0",
    "24": "Nougat 7.0",
    "25": "Nougat 7.1",
    "26": "Oreo 8.0",
    "27": "Oreo 8.1.0",
    "28": "P ",
    "29": "Q"
}

# https://android.jlelse.eu/permission-protection-levels-969c9d0a7ebc
# https://developer.android.com/guide/topics/manifest/permission-element
# 0 - normal | 1 - dangerous | 2 - signature
ANDROID_PERMISSION_PROTECTION_LEVELS = {
    0: "Normal",
    1: "Dangerous",
    2: "Signature"
}

class DroidStatX:
    a = ""
    d = ""
    dx = ""
    xml = ""
    manifest_xml = {}
    app_xml = {}
    exportedActivities = []
    intentFilterList = {}
    componentPermissionList = {}
    activitiesWithExcludeFromRecents = []
    activitiesExtendPreferencesWithValidate = []
    activitiesExtendPreferencesWithoutValidate = []
    activitiesWithoutFlagSecure = []
    exportedReceivers = []
    exportedProviders = []
    exportedServices = []
    permissions = []
    permissionProtectionLevels = {}
    secretCodes = []
    libs = []
    assemblies = []
    assets = []
    cordova = []
    rawResources = []
    dexFiles = []
    otherFiles = []
    cordovaPlugins = []
    networkSecurityConfigDomains = []
    isAppXamarin = False
    xamarinMKBundled = False
    xamarinBundledFile = ""
    isAppCordova = False
    isAppOutsystems = False
    networkSecurityConfig = False
    minSDKVersion = ""
    targetSDKVersion = ""
    versionCode = ""
    versionName = ""
    sha256 = ""
    packageName = ""
    debuggable = False
    allowBackup = False
    certificates = []
    signatureFiles = []

    baksmaliPaths = []
    smaliChecks = None

    def __init__(self, args, apk):
        self.args = args
        self.apk = apk.apk  # androguard.core.bytecodes.apk
        self.sha256 = apk.sha256
        self.output_name = apk.output_name
        self.output_path = apk.output_path

        self.smaliChecks = SmaliChecks(self.args, apk)
        self.performAnalysis()

    def performAnalysis(self):
        log.info("Analysing manifest.xml")
        self.manifest_xml = self.apk.get_android_manifest_axml().get_xml_obj()
        self.app_xml = self.manifest_xml.findall("application")[0]

        log.info("Extracting Activities")
        self.extractActivitiesWithExcludeFromRecents()
        self.extractActivitiesWithoutSecureFlag()
        log.info("Extracting Package Properties")
        self.extractPackageProperties()
        log.info("Extracting Exported Components")
        self.extractExportedComponents()
        log.info("Extracting Permissions")
        self.extractPermissions()
        self.extractPermissionProtectionLevels()
        log.info("Extracting Certificates")
        self.extractCertificates()
        log.info("Extracting Files")
        self.extractFiles()

        self.export_analysis()

    def export_analysis(self):
        data = self.get_analysis()
        content = []
        for elem in sorted(data.items()):
            content.append(elem[0] + ": " + str(elem[1]))
        data = self.smaliChecks.get_analysis()
        for elem in sorted(data.items()):
            content.append(elem[0] + ": " + str(elem[1]))

        filename = self.output_name + ".txt"
        droidsf.utils.export_file(self.args.output_path, filename, content)
        log.info("Exported analysis to: %s", filename)

    # Return the Android Code Name for the particular Api Level.
    def getCodeName(self, apiLevel):
        return ANDROID_CODENAMES[apiLevel]

    # Extract package properties such as the minSDKVersion, PackageName, VersionName, VersionCode, isDebuggable, allowBackup
    def extractPackageProperties(self):
        usesSDK = self.manifest_xml.findall("uses-sdk")
        self.minSDKVersion = self.apk.get_min_sdk_version()
        self.targetSDKVersion = self.apk.get_target_sdk_version()
        self.packageName = self.apk.get_package()
        self.versionName = self.apk.get_androidversion_name()
        self.versionCode = self.apk.get_androidversion_code()
        if self.app_xml.get(NS_ANDROID + "debuggable") == 'true':
            self.debuggable = True
        if self.app_xml.get(NS_ANDROID + "allowBackup") == 'true':
            self.allowBackup = True
        elif self.app_xml.get(NS_ANDROID + "allowBackup") == 'false':
            self.allowBackup = False
        else:
            self.allowBackup = True

        networkSecurityConfig = self.app_xml.get(NS_ANDROID + "networkSecurityConfig")
        if networkSecurityConfig:
            self.networkSecurityConfig = True
            self.parseNetworkSecurityConfigFiles(networkSecurityConfig)

    def parseNetworkSecurityConfigFiles(self, namespace):
        xml_path = os.path.join(self.output_path, "res/xml")
        config_file = os.path.join(xml_path, "network_security_config.xml")

        xml_files = []
        if os.path.isfile(config_file):
            xml_files.append(config_file)
        elif os.path.isdir(xml_path):
            for root, dirs, files in os.walk(xml_path):
                xml_files = [os.path.join(root, f) for f in files if f.endswith(".xml")]
                # for name in files:
                #     xml_files.append(os.path.join(root, name))

        for file_path in xml_files:
            tree = ET.parse(file_path)
            root = tree.getroot()
            if self.parseNetworkSecurityConfigFile(root):
                log.info("Found network security config (%s) file: %s", namespace, file_path)

    def parseNetworkSecurityConfigFile(self, root):
        foundConfig = False

        for child in root:
            if child.tag == "base-config" or child.tag == "domain-config":
                domainConfig = {
                    'domains': [],
                    'allowClearText': True,
                    'allowUserCA': False,
                    'pinning': False,
                    'pinningExpiration': ''
                }
                if 'cleartextTrafficPermitted' in child.attrib:
                    if child.attrib['cleartextTrafficPermitted'] == "false":
                        domainConfig['allowClearText'] = False
                for sub in child:
                    if sub.tag == "domain":
                        domainConfig['domains'].append(sub.text)
                    if sub.tag == "trust-anchors":
                        for certificates in sub:
                            if certificates.attrib['src'] == "user":
                                domainConfig['allowUserCA'] = True
                    if sub.tag == "pin-set":
                        domainConfig['pinning'] = True
                        if 'expiration' in sub.attrib:
                            domainConfig['pinningExpiration'] = sub.attrib['expiration']
                self.networkSecurityConfigDomains.append(domainConfig)
                foundConfig = True

        return foundConfig

    # Create the list of permissions used by the package
    def extractPermissions(self):
        for permission in self.apk.get_permissions():
            # log.debug("Application requires permission: %s", permission)
            self.permissions.append(permission)

    def extractPermissionProtectionLevels(self):
        self.permissionProtectionLevels = {}
        for permission in self.manifest_xml.findall("permission"):
            name = permission.get(NS_ANDROID + "name")
            level = int(permission.get(NS_ANDROID + "protectionLevel"), 16)
            level_name = ANDROID_PERMISSION_PROTECTION_LEVELS[level]

            # log.debug("Found Protection Level for Permission : %s - %s", name, level_name)
            self.permissionProtectionLevels[name] = level_name

    # Extract certificates used and signature file names
    def extractCertificates(self):
        if self.apk.is_signed():
            # Test if signed v1 or v2 or both
            if self.apk.is_signed_v1() and self.apk.is_signed_v2():
                scheme_used = "v1+v2"
            elif self.apk.is_signed_v1():
                scheme_used = "v1"
            else:
                scheme_used = "v2"

        # Iterate over all certificates
        for cert in self.apk.get_certificates():
            # Each cert is now a asn1crypt.x509.Certificate object
            data = {
                "sha1": cert.sha1.hex(),  # the sha1 fingerprint
                "sha256": cert.sha256.hex(),  # the sha256 fingerprint
                "issuer": cert.issuer.human_friendly,  # issuer
                "subject": cert.subject.human_friendly,  # subject, usually the same
                "hash_algo": cert.hash_algo,  # hash algorithm
                "signature_algo": cert.signature_algo,  # Signature algorithm
                "serial_number": cert.serial_number,  # Serial number
                # "contents": cert.contents,  # The DER coded bytes of the certificate itself
            }
            self.certificates.append(data)

        # Return the name of the first signature file found.
        self.signatureFiles = self.apk.get_signature_names()

    # Create a global list of activities with the excludeFromRecents attribute
    # XXX: Checked with sweatcoin
    def extractActivitiesWithExcludeFromRecents(self):
        for activity in self.app_xml.findall("activity"):
            if activity.get(NS_ANDROID + "excludeFromRecents") == 'true':
                self.activitiesWithExcludeFromRecents.append(activity.get(NS_ANDROID + "name"))

    # Create a global list of activities that do not have the FLAG_SECURE or the excludeFromRecents attribute set.
    # XXX: Checked with pt.santandertotta.mobileparticulares
    def extractActivitiesWithoutSecureFlag(self):
        activitiesWithoutSecureFlag = []

        for activity in self.app_xml.findall("activity"):
            activityName = activity.get(NS_ANDROID + "name")

            if self.smaliChecks.doesActivityHasFlagSecure(activityName):
                log.debug("Activity %s sets FLAG_SECURE.", activityName)
            elif activityName not in self.activitiesWithExcludeFromRecents:
                self.activitiesWithoutFlagSecure.append(activityName)
            else:
                log.debug("Activity %s is on ExcludeFromRecents.", activityName)

    # Add the extracted permission of a particular component to a global list indexed by the component name.
    def extractComponentPermission(self, component):
        name = component.get(NS_ANDROID + "name")
        permission = component.get(NS_ANDROID + "permission")

        if permission:
            self.componentPermissionList[name] = permission

    # https://developer.android.com/guide/components/intents-filters
    # Create a global list with that particular object intent-filters indexed to the component name.
    def extractIntentFilters(self, component):
        name = component.get(NS_ANDROID + "name")
        items = component.findall("intent-filter")

        if not items:
            # log.debug("Component %s has no Intent Filters.", name)
            return

        filterList = []
        for item in items:
            intent_filter = {
                "action": [],
                "category": [],
                "data": []
            }
            for action in item.findall("action"):
                intent_filter["action"].append(action.get(NS_ANDROID + "name"))
            for category in item.findall("category"):
                intent_filter["category"].append(category.get(NS_ANDROID + "name"))
            for data in item.findall("data"):
                scheme = data.get(NS_ANDROID + "scheme")
                host = data.get(NS_ANDROID + "host")
                port = data.get(NS_ANDROID + "port")
                path = data.get(NS_ANDROID + "path")
                pathPattern = data.get(NS_ANDROID + "pathPattern")
                pathPrefix = data.get(NS_ANDROID + "pathPrefix")
                mimeType = data.get(NS_ANDROID + "mimeType")
                if scheme:
                    intent_filter["data"].append("scheme:" + scheme)
                    # Check presence of a SECRET_CODE
                    if scheme == "android_secret_code":
                        self.secretCodes.append(name)
                        intent_filter["data"].append("android_secret_code")
                if host:
                    intent_filter["data"].append("host:" + host)
                if port:
                    intent_filter["data"].append("port:" + port)
                if path:
                    intent_filter["data"].append("path:" + path)
                if pathPattern:
                    intent_filter["data"].append("pathPattern:" + pathPattern)
                if pathPrefix:
                    intent_filter["data"].append("pathPrefix:" + pathPrefix)
                if mimeType:
                    intent_filter["data"].append("mimeType:" + mimeType)

            filterList.append(intent_filter)

        self.intentFilterList[name] = filterList

    # Determine exported Activities taking into account the existence of exported attribute or the presence of intent-filters and check if vulnerable to Fragment Injection
    def extractExportedActivities(self):
        for activity in self.app_xml.findall("activity"):
            activityName = activity.get(NS_ANDROID + "name")

            # Extract Permission
            self.extractComponentPermission(activity)

            # Extract Intent Filters
            self.extractIntentFilters(activity)

            # Detect OutSystems applications
            if "com.outsystems.android" in activityName:
                    self.isAppOutsystems = True

            exported = activity.get(NS_ANDROID + "exported")  # 'true' / 'false' / None
            if exported == "true":
                # log.debug("Exported activity %s: found 'exported' attribute.", activityName)
                self.exportedActivities.append(activityName)
            elif exported != "false" and activityName in self.intentFilterList:
                # log.debug("Exported activity %s: found Intent Filters.", activityName)
                self.exportedActivities.append(activityName)

    def analyseExportedActivities(self):
        for activity in self.exportedActivities:
            res = self.smaliChecks.doesPreferenceActivityHasValidFragmentCheck(activity)
            if res is True:
                self.activitiesExtendPreferencesWithValidate.append(activity)
            elif res is False:
                self.activitiesExtendPreferencesWithoutValidate.append(activity)

    # Determine exported Broadcast Receivers taking into account the existence of exported attribute or the presence of intent-filters
    def extractExportedReceivers(self):
        for receiver in self.app_xml.findall("receiver"):
            receiverName = receiver.get(NS_ANDROID + "name")

            # Extract Permission
            self.extractComponentPermission(receiver)

            # Extract Intent Filters
            self.extractIntentFilters(receiver)

            exported = receiver.get(NS_ANDROID + "exported")  # 'true' / 'false' / None
            if exported == "true":
                # log.debug("Exported receiver %s: found 'exported' attribute.", receiverName)
                self.exportedReceivers.append(receiverName)
            elif exported != "false" and receiverName in self.intentFilterList:
                # log.debug("Exported activity %s: found Intent Filters.", receiverName)
                self.exportedReceivers.append(receiverName)

    # Determine exported Content Providers taking into account the existence of exported attribute or without the attributes, under API 16 they are exported by default
    def extractExportedProviders(self):
        for provider in self.app_xml.findall("provider"):
            providerName = provider.get(NS_ANDROID + "name")

            # Extract Permission
            self.extractComponentPermission(provider)

            exported = provider.get(NS_ANDROID + "exported")  # 'true' / 'false' / None
            if exported == "true":
                # log.debug("Exported provider %s: found 'exported' attribute.", providerName)
                self.exportedProviders.append(providerName)
            elif exported != "false" and self.minSDKVersion <= 16:
                # log.debug("Exported provider %s: minSDK version <= 16.", providerName)
                self.exportedProviders.append(providerName + " * In devices <= API 16 (Jelly Bean 4.1.x)")

    def analyseExportedProviders(self):
        for provider in self.exportedProviders:
            self.smaliChecks.determineContentProviderSQLi(provider)
            self.smaliChecks.determineContentProviderPathTraversal(provider)

    # Determine exported Services taking into account the existence of exported attribute or the presence of intent-filters
    def extractExportedServices(self):
        for service in self.app_xml.findall("service"):
            serviceName = service.get(NS_ANDROID + "name")

            # Extract Permission
            self.extractComponentPermission(service)

            # Extract Intent Filters
            self.extractIntentFilters(service)

            exported = service.get(NS_ANDROID + "exported")  # 'true' / 'false' / None
            if exported == "true":
                # log.debug("Exported service %s: found 'exported' attribute.", serviceName)
                self.exportedServices.append(serviceName)
            elif exported != "false" and serviceName in self.intentFilterList:
                # log.debug("Exported service %s: found Intent Filters.", serviceName)
                self.exportedServices.append(serviceName)

    # Run the functions that extract the exported components.
    def extractExportedComponents(self):
        self.extractExportedActivities()
        self.analyseExportedActivities()
        self.extractExportedReceivers()
        self.extractExportedProviders()
        self.analyseExportedProviders()
        self.extractExportedServices()

    # Determine if path is in the exclude paths configured in the config file.
    def isInExclusions(self, filename, exclusions):
        for pattern in exclusions:
            if pattern in filename:
                return True
        return False

    # Create a list of files, organized in several types and while doing it, by the existence of certain files, determine if the app is a Cordova or Xamarin app.
    def extractFiles(self):
        files = self.apk.get_files()
        try:
            for f in files:
                if self.isInExclusions(f, self.args.file_exclusions):
                    continue
                try:
                    f.encode("ascii")
                except UnicodeEncodeError as e:
                    f = f.encode('ascii', 'xmlcharrefreplace')
                if "assets/www/" in f:
                    if "assets/www/cordova.js" in f:
                        self.isAppCordova = True
                    if "assets/www/plugins/" in f:
                        beginPos = f.find("/plugins/") + 9
                        endPos = f.find("/", beginPos)
                        item = f[beginPos:endPos]
                        self.cordovaPlugins.append(item) if item not in self.cordovaPlugins else None
                    self.cordova.append(f)
                elif f[0:4] == "lib/":
                    self.libs.append(f)
                    if 'libmonodroid_bundle_app.so' in f:
                        self.isAppXamarin = True
                        self.xamarinMKBundled = True
                        self.xamarinBundledFile = f
                        self.unbundleXamarinDlls(f)
                elif f[0:11] == "assemblies/" in f:
                    self.assemblies.append(f)
                    if 'Xamarin.' in f and self.isAppXamarin is False:
                        self.isAppXamarin = True
                elif "assets/" in f:
                    self.assets.append(f)
                elif "res/raw/" in f:
                    self.rawResources.append(f)
                elif ".dex" in f:
                    self.dexFiles.append(f)
                else:
                    self.otherFiles.append(f)
        except UnicodeDecodeError as e:
            pass

    def unbundleXamarinDlls(self, lib_path):
        export_path = "xamarin_" + lib_path.split("/")[1]
        file_path = os.path.join(self.output_path, lib_path)
        output_path = os.path.join(self.output_path, export_path)
        log.info("Unpacking %s to %s.", lib_path, output_path)
        with open(file_path, "rb") as f:
            data = f.read()
            elffile = ELFFile(f)
            log.debug("Loaded ELF file: %s", lib_path)
            section = elffile.get_section_by_name('.dynsym')

            for symbol in section.iter_symbols():
                if symbol['st_shndx'] != 'SHN_UNDEF' and symbol.name.startswith('assembly_data_'):
                    start_offset = symbol['st_value']
                    end_offset = symbol['st_value'] + symbol['st_size']

                    dll_data = data[start_offset:end_offset]
                    dll_data = gzip.GzipFile(fileobj=BytesIO(dll_data)).read()

                    filename = symbol.name[14:].replace('_dll', '.dll')
                    droidsf.utils.export_file(output_path, filename, dll_data, mode="wb")
                    log.debug("Exported Xamarin DLL: %s", filename)

    # *** GETTERS ***

    def get_analysis(self):
        attrs = [
            "permissions",
            # "permissionProtectionLevels",
            "exportedActivities",
            "exportedReceivers",
            "exportedProviders",
            "exportedServices",
            "packageName",
            "minSDKVersion",
            "targetSDKVersion",
            "versionName",
            "versionCode",
            "sha256",
            "isAppCordova",
            "isAppXamarin",
            "xamarinMKBundled",
            "isAppOutsystems",
            "debuggable",
            "allowBackup",
            "networkSecurityConfig",
            "isMultiDex",
            "assets",
            "cordova",
            "rawResources",
            "libs",
            "otherFiles",
            "assemblies",
            "dexFiles",
            "secretCodes",
            "intentFilterList",
            "networkSecurityConfigDomains",
            "activitiesWithExcludeFromRecents",
            "activitiesWithoutFlagSecure",
            "activitiesExtendPreferencesWithValidate",
            "activitiesExtendPreferencesWithoutValidate",
            "cordovaPlugins",
            "certificates",
            "signatureFiles"
        ]
        data = {}
        for attr in attrs:
            value = getattr(self, attr)
            if isinstance(value, list):
                try:
                    value = sorted(value)
                except TypeError:
                    pass
            elif callable(value):
                value = value()

            data[attr] = value

        return data

    # Return the permission defined in the particular component.
    # XXX: unused
    def getComponentPermission(self, name):
        try:
            return self.componentPermissionList[name]
        except:
            return ""

    # Return the app permissions global list
    def getPermissions(self):
        return self.permissions

    # Return the exported activities global list
    def getExportedActivities(self):
        return self.exportedActivities

    # Return the the exported broadcast receivers global list
    def getExportedReceivers(self):
        return self.exportedReceivers

    # Return the exported content providers global list
    def getExportedProviders(self):
        return self.exportedProviders

    # Return the exported services global list
    def getExportedServices(self):
        return self.exportedServices

    # Return the app package name
    def getPackageName(self):
        return self.packageName

    # Return the app minSDKVersion
    def getMinSDKVersion(self):
        return self.minSDKVersion

    # Return the app targetSDKVersion
    def getTargetSDKVersion(self):
        return self.targetSDKVersion

    # Return the app versionName
    def getVersionName(self):
        return self.versionName

    # Return the app versionCode
    def getVersionCode(self):
        return self.versionCode

    # Return the APK SHA256
    def getSHA256(self):
        return self.sha256

    def isCordova(self):
        if self.isAppCordova is True:
            return "Yes"
        else:
            return "No"

    def isXamarin(self):
        if self.isAppXamarin is True:
            return "Yes"
        else:
            return "No"

    def isXamarinBundled(self):
        if self.xamarinMKBundled is True:
            return "Yes"
        else:
            return "No"

    def isOutsystems(self):
        if self.isAppOutsystems is True:
            return "Yes"
        else:
            return "No"

    def isDebuggable(self):
        if self.debuggable is True:
            return "Yes"
        else:
            return "No"

    def isBackupEnabled(self):
        if self.allowBackup is True:
            return "Yes"
        else:
            return "No"

    def hasNetworkSecurityConfig(self):
        return self.networkSecurityConfig

    def isMultiDex(self):
        return len(self.dexFiles) > 1

    def getAssets(self):
        return self.assets

    def getCordovaFiles(self):
        return self.cordova

    def getRawResources(self):
        return self.rawResources

    def getLibs(self):
        return self.libs

    def getOtherFiles(self):
        return self.otherFiles

    def getXamarinAssemblies(self):
        return self.assemblies

    def getDexFiles(self):
        return self.dexFiles

    def getSecretCodes(self):
        return self.secretCodes

    def getIntentFiltersList(self):
        return self.intentFilterList

    def getNetworkSecurityConfigDomains(self):
        return self.networkSecurityConfigDomains

    def getActivitiesWithExcludeFromRecents(self):
        return self.activitiesWithExcludeFromRecents

    def getActivitiesWithoutSecureFlag(self):
        return self.activitiesWithoutFlagSecure

    def getActivitiesExtendPreferencesWithValidate(self):
        return self.activitiesExtendPreferencesWithValidate

    def getActivitiesExtendPreferencesWithoutValidate(self):
        return self.activitiesExtendPreferencesWithoutValidate

    def getCordovaPlugins(self):
        return self.cordovaPlugins


"""
Kontur Algılama:
    Kontur tespiti, atnı renk veya yoğunluğa sahip tüm kesintisiz
    noktaları (sınırla birlikte) birleştirmeyi amaçlayan yöntemdir.

    Konturlar şekil analizi ve nesne algılama ve tanıma için kullanılır.

"""
import cv2
import matplotlib.pyplot as plt
import numpy as np

#resmi içe aktar
img = cv2.imread("contour.jpg", 0)
plt.figure()
plt.imshow(img, cmap = "gray")
plt.axis("off")

#kontur tespiti
image, contours, hierarch = cv2.findContours(img, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)

external_contour = np.zeros(img.shape)
imternal_contour = np.zeros(img.shape)

for i in range(len(contours)):

    #external
    if hierarch[0][i][3] == -1:
        cv2.drawContours(external_contour, contours, i, 255, -1)# 255 renk, -1 kalınlık
    else: #internal
        cv2.drawContours(internal_contour, contours, i, 255, -1)
plt.figure()
plt.imshow(external_contour, cmap = "gray")
plt.axis("off")

plt.figure()
plt.imshow(internal_contour, cmap = "gray")
plt.axis("off")

3_OpenCV ile Nesne Tespiti/3_contour_detection/kontur_algilama.py


from django.contrib.auth import login
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm
from django.utils.text import slugify
from django.shortcuts import render, redirect, get_object_or_404
from . import forms

from .models import Vendor
from apps.product.models import Product, ProductImage
from apps.order.models import Order, OrderItem

from . import forms
from django.contrib.auth import login
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm
from django.utils.text import slugify
from django.shortcuts import render, redirect, get_object_or_404
from . import forms

from .models import Vendor
from apps.product.models import Product, ProductImage
from apps.order.models import Order, OrderItem

from . import forms
from django.contrib.auth import login
from django.contrib.auth.decorators import login_required
from django.contrib.auth.forms import UserCreationForm
from django.utils.text import slugify
from django.shortcuts import render, redirect, get_object_or_404

from .models import Vendor
from apps.product.models import Product, ProductImage

from .forms import ProductForm, ProductImageForm

from django.contrib import messages
from django.db.models import Q
from django.shortcuts import render, get_object_or_404, redirect

from apps.product.forms import AddToCartForm
from apps.product.models import Category, Product

from apps.cart.cart import Cart

def vendors(request):
    vendors = Vendor.objects.all()

    return render(request, 'vendor/chef.html', {'vendors': vendors})

def vendor(request, vendorName):
    vendor = get_object_or_404(Vendor, pk=vendorName)
    cart = Cart(request)
    imagesstring = '{"thumbnail": "%s","image": "%s", "id": "mainimage"},' % (
        vendor.get_thumbnail(), vendor.logo.url)
    print(imagesstring)
    if request.method == 'POST':
        form = AddToCartForm(request.POST)

        if form.is_valid():
            quantity = 1
            # product.id =1
            product_id = form.cleaned_data['product_id']

            # form.cleaned_data['quantity']

            cart.add(product_id=product_id, quantity=quantity, update_quantity=False)

            messages.success(request, 'The product was added to the cart')

    else:
        form = AddToCartForm()

    return render(request, 'vendor/chef.html', {'form': form ,'vendor': vendor,'imagesstring': "[" + imagesstring.rstrip(',') + "]"})

def become_chef(request):
    if request.method == 'POST':
        form = UserCreationForm(request.POST)

        if form.is_valid():
            user = form.save()

            login(request, user)

            chef = Vendor.objects.create(name=user.username, created_by=user)

            return redirect("vendor:chef-order")
    else:
        form = UserCreationForm()

    return render(request, 'chef/sign_up.html', {'form': form})

@login_required(login_url='/chef/sign-in/')
def chef_admin(request):
    return redirect("vendor:chef-order")

@login_required(login_url='/chef/sign-in/')
def chef_order(request):
    # if request.method == "POST":
        # order = Order.objects.get(id=request.POST["id"],
        #                           vendor=request.user.vendor)

        # if order.status == Order.COOKING:
        #     order.status = Order.READY
        #     order.save()
    chef = request.user.vendor
    orders = chef.orders.all()
    # #TODO: fix filter

    return render(request, 'chef/order.html', {'orders': orders,'orders': orders, 'chef': chef})

@login_required(login_url='/chef/sign-in/')
def chef_meal(request):
    chef = request.user.vendor
    meals = Product.objects.filter(
        vendor=chef).order_by("-id")
    return render(request, 'chef/meal.html', {"meals": meals, 'chef': chef}) #

@login_required(login_url='/chef/sign-in/')
def chef_customers(request):
    return redirect('/')
# @login_required(login_url='/chef/sign-in/')
# def chef_customers(request):

#     customers = Customer.objects.annotate(total_order=Count(
#         Case(When(order__chef=request.user.chef,
#                   then=1)))).order_by("-total_order")

#     all_customers = [
#         customer for customer in customers if customer.total_order > 0
#     ]

#     return render(request, 'chef/customers.html',
#                   {"all_customers": all_customers})

@login_required(login_url='/chef/sign-in/')
def chef_account(request):
    user_form = forms.UserFormForEdit(instance=request.user)
    chef_form = forms.ChefForm(instance=request.user)
    chef = request.user.vendor

    if request.method == "POST":
        user_form = forms.UserFormForEdit(request.POST, instance=request.user)
        chef_form = forms.ChefForm(request.POST,
                                   request.FILES,
                                   instance=request.user.vendor)

        if user_form.is_valid() and chef_form.is_valid():
            user_form.save()
            chef_form.save()

    return render(request, 'chef/account.html', {
        "user_form": user_form,
        'chef': chef,
        "chef_form": chef_form #TODO: change forms to accomodate the model used
# TODO: hook it up, take notes from edit_vendor
    })

@login_required(login_url='/chef/sign-in/')
def chef_add_meal(request):
    chef = request.user.vendor
    if request.method == 'POST':
        form = forms.ProductForm(request.POST, request.FILES)

        if form.is_valid():
            product = form.save(commit=False)
            product.vendor = request.user.vendor
            product.slug = slugify(product.title)
            product.save()

            return redirect("vendor:chef-meal")
    else:
        form = forms.ProductForm()

    return render(request, 'chef/add_meal.html', {
        'chef': chef,
        'form': form
    })
@login_required(login_url='/chef/sign-in/')
def chef_report(request):
    return render(request, 'chef/report.html', {}) # TODO: get the actual report to work

@login_required(login_url='/chef/sign-in/')
def chef_edit_meal(request, meal_id):
    form = forms.ProductForm(instance=Product.objects.get(id=meal_id))

    if request.method == "POST":
        form = forms.ProductForm(request.POST,
                        request.FILES,
                        instance=Product.objects.get(id=meal_id))
        if form.is_valid():
            form.save()
            return redirect(chef_meal)

    return render(request, 'chef/edit_meal.html', {"form": form}) # TODO: hook it up, take notes from edit_product


"""
Obtains the N most abundant bugs from a study object and saves it as a study
"""
import mdsine2 as md2
import numpy as np
import argparse
from pathlib import Path
import pandas as pd

def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("-f", "--study_file", required=True, type=str,
                        help="<Required> The location of the study object")
    parser.add_argument("-o", "--output_path", required=True, type=str,
                        help="<Required> Path to which the output should be saved to.")
    parser.add_argument("-n", "--n_bugs", required=True, type=int,
                        help="<Required> The top n most abundant bugs that are retained")

    return parser.parse_args()

def count_times(df, times, times_cnts, t2idx):
    """
    count the number of times data is collected at a given sample time point
    """

    for col in df.columns:
        if col in times:
            times_cnts[t2idx[col]] += 1

    return times_cnts

def add_unequal_col_dataframes(df, dfother, times, times_cnts, t2idx):
    """combine 2 dataframe that may have different columns headings"""

    times_cnts = count_times(dfother, times, times_cnts, t2idx)
    if df is None:
        return dfother, times_cnts

    cols_toadd_df = []
    cols_toadd_dfother = []
    for col in dfother.columns:
        if col not in df.columns:
            cols_toadd_df.append(col)
    for col in df.columns:
        if col not in dfother.columns:
            cols_toadd_dfother.append(col)

    df = pd.concat([df,
                    pd.DataFrame(np.zeros(shape=(len(df.index), len(cols_toadd_df))),
                                 index=df.index, columns=cols_toadd_df)], axis=1)
    dfother = pd.concat([dfother,
                         pd.DataFrame(np.zeros(shape=(len(dfother.index), len(cols_toadd_dfother))),
                                      index=dfother.index, columns=cols_toadd_dfother)], axis=1)

    return dfother.reindex(df.index) + df, times_cnts

def combine_dfs_subject(subjset, dtype):
    """
       combines the abundance of OTUs in different subjects present in subjset
       together in a DataFrame
    """

    times = []
    for subj in subjset:
        times = np.append(times, subj.times)
    times = np.sort(np.unique(times))
    t2idx = {}
    for i, t in enumerate(times):
        t2idx[t] = i
    times_cnts = np.zeros(len(times))

    df = None
    for subj in subjset:
        dfnew = subj.df()[dtype]
        df, times_cnts = add_unequal_col_dataframes(df=df, dfother=dfnew,
                                                    times=times, times_cnts=times_cnts, t2idx=t2idx)
    df = df / times_cnts
    return df

def get_top_abundance_data(study, N):
    """
       makes a study object consisting of the N most abundant OTUs
       @parameters
       -----------------------
       subjset : mdsine2.base.Study

       @returns
       -----------------------
       mdsine2.base.study
    """

    combined_df = combine_dfs_subject(study, "abs")
    df_np = combined_df.to_numpy()
    total_abund = df_np.sum(axis=1)
    idxs = np.argsort(total_abund)[::-1]
    all_taxa = study.taxa

    taxa_to_remove = [all_taxa[i].name for i in idxs[N:]]

    new_study = study.pop_taxa(taxa_to_remove)
    retained_taxa = [t.name for t in new_study.taxa]
    print("Retained Taxa: {}".format(retained_taxa))

    return new_study

if __name__ == "__main__":
    args = parse_arguments()
    study = md2.Study.load(args.study_file)
    top_otu_study = get_top_abundance_data(study, args.n_bugs)
    save_path = Path(args.output_path)
    save_path.parent.mkdir(parents=True, exist_ok=True)

    top_otu_study.save(save_path)
    print("Saved study object for top {} OTUs".format(args.n_bugs))


import hashlib
from datetime import datetime

BLOCK_STRING_FORMAT = "{0};{1};{2};{3}"

def calculate_hash(data):
  sha = hashlib.sha256()

  sha.update(data.encode('utf-8'))

  return sha.hexdigest()

class Block:

  def __init__(self, timestamp, data, previous_hash):
    self.timestamp     = timestamp
    self.data          = data
    self.previous_hash = previous_hash

    hashable_data = self.data + str(self.timestamp)

    self.hash = calculate_hash(hashable_data)
    self.next = None

  def get_hash(self):
    return self.hash

  def get_next(self):
    return self.next

  def set_next(self, block):
    self.next = block

  def __repr__(self):
    return BLOCK_STRING_FORMAT.format(str(self.timestamp), self.data, self.hash, self.previous_hash)

class Blockchain:
  def __init__(self):
    self.head = None
    self.tail = None

  def add(self, data):
    if data:
      timestamp = datetime.utcnow()

      if self.head == None:
        new_block = Block(timestamp, data, None)
        self.head = new_block
        self.tail = self.head
      else:
        new_block = Block(timestamp, data, self.tail.get_hash())
        self.tail.set_next(new_block)
        self.tail = new_block

  def to_list(self):
    output_list   = []
    current_block = self.head

    while current_block:
        output_list.append(current_block)
        current_block = current_block.get_next()

    return output_list

#Test 1. Add 4 blocks with valid data.
blockchain_test1 = Blockchain()

blockchain_test1.add("Block 0")
blockchain_test1.add("Block 1")
blockchain_test1.add("Block 2")
blockchain_test1.add("Block 3")

#Expected: Prints the whole blockchain, with 4 blocks.
print("TEST 1 OUTPUT: \n", blockchain_test1.to_list())

#Test 2. Print the blockchain when no blocks were added.
blockchain_test2 = Blockchain()

#Expected: Prints an empty list.
print("TEST 2 OUTPUT: \n", blockchain_test2.to_list())

#Test 3. Call the add() function with None and an empty string as parameter.
blockchain_test3 = Blockchain()

blockchain_test3.add(None)
blockchain_test3.add("")

#Expected: Prints an empty list.
print("TEST 3 OUTPUT: \n", blockchain_test3.to_list())


from django.urls import path
from . import views

urlpatterns = [
    path('community/', views.community, name="community"),
    path('post/', views.post, name="post"),
    path('community/delete/<int:id>', views.post_delete, name="post_delete"),
    path('community/<int:id>', views.post_detail, name="post_detail"),
    path('community/post/update/<int:id>', views.update, name="update"),
    path('community/comment/<int:id>', views.write_comment, name="write_comment"),
    path('community/comment/update/<int:id>', views.comment_update, name="comment_update"),
    path('community/comment/delete/<int:id>', views.delete_comment, name="delete_comment"),
    path('like/<int:post_id>', views.like, name="likes"),
    path('community/comment/like/<int:id>', views.comment_like, name="comment_like"),
    path('tag/', views.TagCloudTV.as_view(), name='tag_cloud'),
    path('tag/<str:tag>/', views.TaggedObjectLV.as_view(), name='tagged_object_list'),
]


import time
from datetime import timedelta, datetime
from lib.sources.Source4Dmoon import *

for x in range(0, 10):
    date = datetime.datetime.strftime(datetime.datetime.now() - timedelta(1) - timedelta(x), '%Y-%m-%d')
    print("############### {} #################".format(date))
    s = Source4Dmoon({
        'resource': '4dmoon',
        'type': 'ma',
        'area': 'ma',
        'url': 'past-results/{}'.format(date),
        'date': date
    })
    s.handle()
    # time.sleep(1)


from flamejam import app, db, mail
from flamejam.utils import get_slug
from flamejam.models.jam import Jam, JamStatusCode
from flamejam.models.game import Game
from flamejam.models.comment import Comment
from flamejam.models.gamepackage import GamePackage
from flamejam.models.gamescreenshot import GameScreenshot
from flamejam.models.rating import Rating
from flamejam.models.rating import RATING_CATEGORIES
from flamejam.forms import (
    WriteComment,
    GameEditForm,
    GameAddScreenshotForm,
    GameAddPackageForm,
    GameCreateForm,
    RateGameForm,
)
from flask import render_template, url_for, redirect, flash, request, abort
from flask_login import login_required, current_user

@app.route("/jams/<jam_slug>/create-game/", methods=["GET", "POST"])
@login_required
def create_game(jam_slug):
    jam = Jam.query.filter_by(slug=jam_slug).first_or_404()

    r = current_user.get_participation(jam)
    if not r or not r.team:
        flash("You cannot create a game without participating in the jam.", category="error")
        return redirect(jam.url())
    if r.team.game:
        flash("You already have a game.")
        return redirect(r.team.game.url())

    enabled = (JamStatusCode.RUNNING <= jam.get_status().code <= JamStatusCode.PACKAGING)

    form = GameCreateForm(request.form, obj=None)
    if enabled and form.validate_on_submit():
        game = Game(r.team, form.title.data)
        db.session.add(game)
        db.session.commit()
        return redirect(url_for("edit_game", jam_slug=jam_slug, game_id=game.id))

    return render_template("jam/game/create.html", jam=jam, enabled=enabled, form=form)

@app.route("/jams/<jam_slug>/<game_id>/edit/", methods=["GET", "POST"])
@login_required
def edit_game(jam_slug, game_id):
    jam = Jam.query.filter_by(slug=jam_slug).first_or_404()
    game = Game.query.filter_by(is_deleted=False, id=game_id).first_or_404()

    if not game or current_user not in game.team.members:
        abort(403)

    form = GameEditForm(request.form, obj=game)
    package_form = GameAddPackageForm()
    screenshot_form = GameAddScreenshotForm()

    if form.validate_on_submit():
        game.title = form.title.data
        game.description = form.description.data
        game.technology = form.technology.data
        game.help = form.help.data

        if game.jam.get_status().code < 4:
            for c in RATING_CATEGORIES:
                setattr(game, "score_{c}_enabled", form.get(c).data)

        game.slug = get_slug(game.title)
        db.session.commit()
        flash("Your settings have been applied.", category="success")
        return redirect(game.url())

    if package_form.validate_on_submit():
        s = GamePackage(game, package_form.url.data, package_form.type.data)
        db.session.add(s)
        db.session.commit()
        flash("Your package has been added.", "success")
        return redirect(request.url)

    if screenshot_form.validate_on_submit():
        s = GameScreenshot(screenshot_form.url.data, screenshot_form.caption.data, game)
        db.session.add(s)
        db.session.commit()
        flash("Your screenshot has been added.", "success")
        return redirect(request.url)

    return render_template("jam/game/edit.html", jam=jam, game=game,
                           form=form, package_form=package_form, screenshot_form=screenshot_form)

@app.route('/edit/package/<id>/<action>/')
@login_required
def game_package_edit(id, action):
    if action not in ("delete"):
        abort(404)

    p = GamePackage.query.filter_by(id=id).first_or_404()
    if current_user not in p.game.team.members:
        abort(403)

    if action == "delete":
        db.session.delete(p)
    db.session.commit()
    return redirect(url_for("edit_game", jam_slug=p.game.jam.slug, game_id=p.game.id))

@app.route('/edit/screenshot/<id>/<action>/')
@login_required
def game_screenshot_edit(id, action):
    if action not in ["up", "down", "delete"]:
        abort(404)

    s = GameScreenshot.query.filter_by(id=id).first_or_404()
    if current_user not in s.game.team.members:
        abort(403)

    if action == "up":
        s.move(-1)
    elif action == "down":
        s.move(1)
    elif action == "delete":
        db.session.delete(s)
        i = 0
        for x in s.game.screenshotsOrdered:
            x.index = i
            i += 1
    db.session.commit()
    return redirect(url_for("edit_game", jam_slug=s.game.jam.slug, game_id=s.game.id))

@app.route('/jams/<jam_slug>/<game_id>/', methods=["POST", "GET"])
def show_game(jam_slug, game_id):
    comment_form = WriteComment()
    jam = Jam.query.filter_by(slug=jam_slug).first_or_404()
    game = Game.query.filter_by(is_deleted=False, id=game_id).filter_by(jam=jam).first_or_404()

    if current_user.is_authenticated and comment_form.validate_on_submit():
        comment = Comment(comment_form.text.data, game, current_user)
        db.session.add(comment)
        db.session.commit()

        # notify the team
        for user in game.team.members:
            if user.notify_game_comment:
                body = render_template("emails/comment.txt", recipient=user, comment=comment)
                mail.send_message(subject=f"{current_user.username} commented on {game.title}",
                                  recipients=[user.email], body=body)

        flash("Your comment has been posted.", "success")
        return redirect(game.url())

    rating = Rating.query.filter_by(game_id=game.id, user_id=current_user.get_id()).first()
    return render_template('jam/game/info.html', game=game, form=comment_form, rating=rating)

@app.route("/jams/<jam_slug>/<game_id>/rate/", methods=["GET", "POST"])
@login_required
def rate_game(jam_slug, game_id):
    jam = Jam.query.filter_by(slug=jam_slug).first_or_404()
    game = Game.query.filter_by(jam_id=jam.id, is_deleted=False, id=game_id).first_or_404()

    form = RateGameForm()
    if jam.get_status().code != JamStatusCode.RATING:
        flash("This jam is not in the rating phase. Sorry, but you cannot rate right now.",
              "error")
        return redirect(game.url())

    if current_user in game.team.members:
        flash("You cannot rate on your own game. Go rate on one of these!", "warning")
        return redirect(url_for("jam_games", jam_slug=jam.slug))

    # Allow only users who participate in this jam to vote.
    if current_user not in jam.participants:
        flash("You cannot rate on this game. Only participants are eligible for vote.", "error")
        return redirect(url_for("jam_games", jam_slug=jam.slug))

    rating = Rating.query.filter_by(game_id=game.id, user_id=current_user.id).first()
    if rating:
        flash("You are editing your previous rating of this game.", "info")

    if form.validate_on_submit():
        new = rating is None
        if not rating:
            rating = Rating(game, current_user, form.note.data, form.score.data)
            db.session.add(rating)
        else:
            rating.text = form.note.data

        for c in ["overall"] + game.ratingCategories:
            rating.set(c, form.get(c).data)

        db.session.commit()
        flash("Your rating has been " + ("submitted" if new else "updated") + ".", "success")
        return redirect(url_for("jam_games", jam_slug=jam.slug))

    elif rating:
        for c in ["overall"] + game.ratingCategories:
            form.get(c).data = rating.get(c)
        form.note.data = rating.text

    return render_template('jam/game/rate.html', jam=jam, game=game, form=form)


import asyncio
from collections.abc import AsyncIterator, Callable
from dataclasses import replace
from datetime import timedelta

import pytest

from platform_disk_api.config import KubeConfig
from platform_disk_api.kube_client import KubeClient
from platform_disk_api.service import DiskNotFound, DiskRequest, Service
from platform_disk_api.usage_watcher import (
    utc_now,
    watch_disk_usage,
    watch_lifespan_ended,
)
from tests.integration.kube import KubeClientForTest

class TestUsageWatcher:
    @pytest.fixture
    def service(self, kube_client: KubeClient, k8s_storage_class: str) -> Service:
        return Service(
            kube_client=kube_client,
            storage_class_name=k8s_storage_class,
        )

    @pytest.fixture
    async def watcher_task(
        self,
        kube_config: KubeConfig,
        kube_client_factory: Callable[[KubeConfig], KubeClientForTest],
        service: Service,
    ) -> AsyncIterator[None]:
        kube_config = replace(kube_config, client_watch_timeout_s=1)  # Force reloads
        async with kube_client_factory(kube_config) as kube_client:
            task = asyncio.create_task(watch_disk_usage(kube_client, service))
            await asyncio.sleep(0)  # Allow task to start
            yield
            task.cancel()
            try:
                await task
            except asyncio.CancelledError:
                pass

    @pytest.fixture
    async def cleanup_task(self, service: Service) -> AsyncIterator[None]:
        task = asyncio.create_task(watch_lifespan_ended(service, check_interval=0.1))
        await asyncio.sleep(0)  # Allow task to start
        yield
        task.cancel()
        try:
            await task
        except asyncio.CancelledError:
            pass

    async def test_usage_watcher_updates_label(
        self,
        watcher_task: None,
        kube_client: KubeClientForTest,
        service: Service,
    ) -> None:
        async def wait_for_last_usage(disk_id: str) -> None:
            while True:
                disk = await service.get_disk(disk_id)
                if disk.last_usage is not None:
                    break
                await asyncio.sleep(0.1)

        for _ in range(10):
            disk = await service.create_disk(
                DiskRequest(1024**2, project_name="test-project"), "user"
            )
            before_start = utc_now()
            async with kube_client.run_pod([disk.id]):
                await asyncio.wait_for(wait_for_last_usage(disk.id), timeout=10)

            disk = await service.get_disk(disk.id)
            assert disk.last_usage
            assert before_start < disk.last_usage

    async def test_task_cleanuped_no_usage(
        self,
        cleanup_task: None,
        service: Service,
    ) -> None:
        disk = await service.create_disk(
            DiskRequest(
                storage=1000,
                life_span=timedelta(seconds=1),
                project_name="test-project",
            ),
            "user",
        )
        await asyncio.sleep(1.5)
        with pytest.raises(DiskNotFound):
            await service.get_disk(disk.id)

    async def test_task_cleanuped_with_usage(
        self,
        cleanup_task: None,
        service: Service,
    ) -> None:
        disk = await service.create_disk(
            DiskRequest(
                storage=1000,
                life_span=timedelta(seconds=2),
                project_name="test-project",
            ),
            "user",
        )
        await asyncio.sleep(1.33)
        await service.mark_disk_usage(disk.id, utc_now())
        await asyncio.sleep(1.33)
        assert await service.get_disk(disk.id)
        await asyncio.sleep(1.33)
        with pytest.raises(DiskNotFound):
            await service.get_disk(disk.id)


import numpy as np
from src.utils_functions import StrToFunc
import sys
import os
from tqdm import tqdm
from src.utils import l_infty_error as lie
from src.get_dataset import get_data
from src.utils import sort_descending as sd
import pickle
import matplotlib.pyplot as plt
from src.utils import sorter

def looper(V, Vlow, Vhigh, xaxis, range_v1, \
           range_v2, filepath, titleval, method, labels="k"):
    fig = plt.figure()
    plt.gcf().clf()
    # fix v1s plot v2s
    # use v1s as labels
    for i in range(len(range_v1)):
        fix_v1 = range_v1[i]
        if len(range_v2) == 1:
            # just a single point, so makes no sense plotting,
            # come back with reversed agruments
            return None
        else:
            xvals, yvals = sorter(xaxis[i,:], V[i,:])
            plt.plot(xvals, yvals, label=labels+"="+str(float("{:.5f}".format(fix_v1))))
            xvals, y1vals, y2vals = sorter(xaxis[i,:], Vlow[i,:], Vhigh[i,:])
            plt.fill_between(xvals, y1vals, y2vals, alpha=0.2)
    plt.xlabel("log matvecs")
    plt.ylabel("log absolute errors")
    plt.legend()
    plt.title("max eigval="+str(titleval))
    plt.savefig(os.path.join(filepath, "grid_search_"+method+" "+labels+".pdf"))
    return None

def plotter(dataset, plot_vals, mode="fix_ks"):
    dir_ = "figures/"
    filedir = dataset
    filepath = os.path.join(dir_, filedir, mode)
    if not os.path.isdir(filepath):
        os.makedirs(filepath)
    if mode == "fix_ks":
        looper(
               plot_vals["mean_log_lies"],\
               plot_vals["p20_log_lies"],\
               plot_vals["p80_log_lies"],\
               plot_vals["log_matvecs"],\
               plot_vals["ks"],\
               plot_vals["lrs"],\
               filepath, plot_vals["max_abs_eigval"],\
               method=plot_vals["method"],\
               labels = "k"
              )
        pass
    if mode == "fix_lrs":
        looper(
               plot_vals["mean_log_lies"].T,\
               plot_vals["p20_log_lies"].T,\
               plot_vals["p80_log_lies"].T,\
               plot_vals["log_matvecs"].T,\
               plot_vals["lrs"],\
               plot_vals["ks"],\
               filepath, plot_vals["max_abs_eigval"],\
               method=plot_vals["method"],\
               labels = "lr"
              )
        pass
    return None

def processor(dataset, outputs, params):
    eps = 1e-32
    # get true eigenvalues for the search ranks
    true_spectrum = outputs["true_spectrum"][params["search_rank"]]
    # get the max magnitude eigenvalue
    max_abs_eigval = max(np.abs(true_spectrum[0]), np.abs(true_spectrum[-1]))
    # expand dims to match shape of approx_eigvals
    true_spectrum = np.expand_dims(true_spectrum, axis=(0,1,2))
    # compute log absolute errors
    abs_errors = np.abs(outputs["approx_eigvals"] - true_spectrum) / max_abs_eigval
    log_lies = np.log(np.max(abs_errors, axis=3)+eps)
    # now compute avg and percentiles
    mean_log_lies = np.mean(log_lies, axis=0)
    p20_log_lies = np.percentile(log_lies, q=20, axis=0)
    p80_log_lies = np.percentile(log_lies, q=80, axis=0)
    log_matvecs = np.log(outputs["matvecs"]+eps)

    # values to send to the plotter function
    plot_vals = {
                 "mean_log_lies": mean_log_lies,
                 "p20_log_lies": p20_log_lies,
                 "p80_log_lies": p80_log_lies,
                 "log_matvecs": log_matvecs,
                 "max_abs_eigval": max_abs_eigval,
                 "ks": params["ks"],
                 "lrs": params["lrs"],
                 "method": params["method"]
                }

    # save these in a file so as to avoid recomputing
    save_dict = {"plt_vals": plot_vals, "outputs": outputs, "params": params}
    dir_path = os.path.join("results", dataset)
    if not os.path.isdir(dir_path):
        os.makedirs(dir_path)
    file_path = os.path.join(dir_path, "grid_search_"+params["method"]+".pkl")
    with open(file_path, "wb") as f:
        pickle.dump(save_dict, f)

    # plot: fix lrs
    plotter(dataset, plot_vals, mode="fix_lrs")
    return None

def computer(dataset, params):
    true_mat, dataset_size, _, _ = get_data(dataset)
    true_spectrum, _ = np.linalg.eig(true_mat)
    true_spectrum = sd(np.real(true_spectrum))

    method = StrToFunc(params["method"])

    ts = params["trials"]
    ks = len(params["ks"])
    lrs = len(params["lrs"])
    srs = len(params["search_rank"])
    ite = params["iters"][0]

    outputs = {"approx_eigvals": np.zeros((ts, ks, lrs, srs)),
               "matvecs": np.zeros((ks, lrs))}
    for t in tqdm(range(params["trials"])):
        for i in range(len(params["ks"])):
            k = params["ks"][i]
            for j in range(len(params["lrs"])):
                lr = params["lrs"][j]
                eigvals, matvecs = method(true_mat, \
                                          k=k, iters=ite, eta=lr,\
                                          sr=params["search_rank"])
                outputs["approx_eigvals"][t,i,j,:] = eigvals
                outputs["matvecs"][i,j] = matvecs
    outputs["true_spectrum"] = true_spectrum[params["search_rank"]]
    return outputs

def main():
    # general setup
    dataset = sys.argv[1]
    trials = int(sys.argv[2])

    # set up parameters
    sr = [0,1,2,3,4,-5,-4,-3,-2,-1]
    ks = list(range(10,100,20))
    iters = [20]#list(range(0,20,2))
    lr = list(np.arange(1e4, 1e7, 1e4))
    algo_params = {
                    "ks": ks, "iters": iters,\
                    "trials": trials,\
                    "search_rank": sr,\
                    "method": "eg_unldd",\
                    "lrs": lr
                  }

    outputs = computer(dataset, algo_params)
    print(algo_params, outputs)
    processor(dataset, outputs, algo_params)
    return None

if __name__ == "__main__":
    main()


from datetime import datetime

import gym
import numpy as np
import torch
import random

from torch import float32, int64

from classes.Agents import AcAgent
from classes.Memory import ReplayMemory
from classes.Models import ValueNet, PolicyNet
from drawing import plot_result_frames
from multiprocessing import Manager, Pool

seed = 333  # np.random.randint(10_000)

device = torch.device('cpu')  # torch.device('cuda' if torch.cuda.is_available() else 'cpu')  #
device_name = 'cpu'  # torch.cuda.get_device_name(device=device) if torch.cuda.is_available() else '-'  #
# env
env_name = 'LunarLander-v2'  # LunarLander-v2 CartPole-v1
env_actions = 4
env_state_dim = 8
# params
lr = 0.0001
hidden = 256
gamma = 0.99
max_frames = 10_000
avg_frames = 1000
batch_size = 64
processes = 3
agents = 3

update_frequency = 32  # 32
rho = 1.0
elastic_type = 'max'  # max, Dkl

def to_tensor(x, dtype=float32):
    return torch.as_tensor(x, dtype=dtype).to(device)

def distance_max(states, model, master_model):
    return torch.max(torch.abs(master_model(states) - model(states)))

def divergence(states, model, master_model):
    p = master_model(states)
    q = model(states)
    return torch.sum(p * torch.log(p / q))

def print_progress(agent_number, message):
    if agent_number == 0:
        print(message)

def main(agent_number, global_value_net, global_policy_net, update_lock):
    local_seed = agent_number + seed
    torch.manual_seed(local_seed)
    torch.cuda.manual_seed(local_seed)
    torch.cuda.manual_seed_all(local_seed)
    np.random.seed(local_seed)
    random.seed(local_seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

    value_net = ValueNet(env_state_dim, hidden).to(device)
    policy_net = PolicyNet(env_state_dim, hidden, env_actions).to(device)

    tmp_value_net = ValueNet(env_state_dim, hidden).to(device)
    tmp_policy_net = PolicyNet(env_state_dim, hidden, env_actions).to(device)

    value_net.load_state_dict(global_value_net.state_dict())
    policy_net.load_state_dict(global_policy_net.state_dict())

    memory = ReplayMemory()
    env = gym.make(env_name)
    agent = AcAgent(env.action_space, policy_net, local_seed)

    score = np.zeros(max_frames)
    final = False
    current_score = 0
    prev_score = 0
    episodes = 0
    state = env.reset(seed=local_seed)[0]

    for t in range(max_frames):
        if final:
            prev_score = current_score
            current_score = 0
            episodes += 1
            state = env.reset()[0]
        action = agent.get_action(state)
        next_state, reward, final, _, _ = env.step(action)
        memory.push(state, action, next_state, reward, final)
        state = next_state

        current_score += 1
        score[t] = prev_score

        if len(memory) > batch_size:
            batch = memory.sample(batch_size)
            states = to_tensor(batch.state)
            actions = to_tensor(batch.action, dtype=int64)
            rewards = to_tensor(batch.reward)
            next_states = to_tensor(batch.next_state)
            finals = to_tensor(batch.final)

            # 1-step Actor-Critic
            # ----------- gradient step 1: CRITIC
            advance = rewards + (1 - finals) * gamma * value_net(next_states) - value_net(states)
            critic_loss = (1 / 2 * advance ** 2).mean()
            critic_grad = torch.autograd.grad(critic_loss, value_net.parameters())
            with torch.no_grad():
                for param, param_grad in zip(value_net.parameters(), critic_grad):
                    param.copy_(param - lr * param_grad)
            # ----------- gradient step 2: ACTOR
            indices = torch.stack((actions, actions))
            act_prob = policy_net(states).T.gather(0, indices)[0]
            actor_loss = (- advance * torch.log(act_prob)).mean()
            actor_grad = torch.autograd.grad(actor_loss, policy_net.parameters())
            with torch.no_grad():
                for param, param_grad in zip(policy_net.parameters(), actor_grad):
                    param.copy_(param - lr * param_grad)

            if t % update_frequency == 0:  # ---------------- elastic update:
                with update_lock:
                    tmp_value_net.load_state_dict(global_value_net.state_dict())
                    tmp_policy_net.load_state_dict(global_policy_net.state_dict())
                    if elastic_type == 'max':
                        distance_value = distance_max(states, value_net, tmp_value_net)
                        distance_policy = distance_max(states, policy_net, tmp_policy_net)
                    elif elastic_type == 'Dkl':
                        distance_value = divergence(states, value_net, tmp_value_net)
                        distance_policy = divergence(states, policy_net, tmp_policy_net)
                    grad_dist_value = torch.autograd.grad(distance_value, value_net.parameters())
                    grad_dist_policy = torch.autograd.grad(distance_policy, policy_net.parameters())
                    elasticity_v = 0 if torch.isnan(distance_value) else lr * critic_loss/distance_value * rho
                    elasticity_p = 0 if torch.isnan(distance_policy) else lr * actor_loss/distance_policy * rho
                    with torch.no_grad():
                        for param, master_param, grad_distance in zip(value_net.parameters(),
                                                                      global_value_net.parameters(), grad_dist_value):
                            param.copy_(param - elasticity_v * grad_distance)
                            master_param.copy_(master_param + elasticity_v * grad_distance)

                        for param, master_param, grad_distance in zip(policy_net.parameters(),
                                                                      global_policy_net.parameters(), grad_dist_policy):
                            param.copy_(param - elasticity_p * grad_distance)
                            master_param.copy_(master_param + elasticity_p * grad_distance)
                    print_progress(agent_number, f"score: {prev_score}, C-loss: {critic_loss}, A-loss: {actor_loss}, "
                                   f"elasticity_v: {elasticity_v}, elasticity_p: {elasticity_p}, "
                                   f"distance_v: {distance_value}, distance_p: {distance_policy}")
    env.close()
    return episodes, score

# https://google.github.io/styleguide/pyguide.html#317-main
if __name__ == '__main__':
    print(f"MULTIPROCESSING AC, processes: {processes}")
    print(f"SEED: {seed}")
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

    global_value_net = ValueNet(env_state_dim, hidden).to(device)
    global_policy_net = PolicyNet(env_state_dim, hidden, env_actions).to(device)

    with Manager() as manager, Pool(processes=processes) as pool:
        update_lock = manager.Lock()

        print(f"------------------------------------ started: {datetime.now().strftime('%Y.%m.%d %H-%M-%S')}")
        pool_args = [(agent, global_value_net, global_policy_net, update_lock) for agent in range(agents)]
        agent_results = pool.starmap(main, pool_args)  # [(episodes, score), (episodes, score)]
        print(f"------------------------------------ finished: {datetime.now().strftime('%Y.%m.%d %H-%M-%S')}")

        episodes = [result[0] for result in agent_results]
        scores = [result[1] for result in agent_results]
        print(f"played episodes: {episodes}")

        title = f'AC Stat. Elastic {agents} agents (asynch)\n' \
                f'update_frequency: {update_frequency}, ' \
                f'stat_elasticity: lr*loss/dist, \n' \
                f'hidden: {hidden}(selu), ' \
                f'batch: {batch_size}, ' \
                f'lr: {lr}, ' \
                f'gamma: {gamma}, ' \
                f'softmax, ' \
                f'seed: {seed}'
        timestamp = datetime.now().strftime("%Y.%m.%d %H-%M-%S")
        filename = f'./output/ac_elastic/{timestamp}_ac_stat_elastic_{agents}.png'
        plot_result_frames(scores, epsilon=None, title=title, info=None,
                           filename=filename, lr=lr, mean_window=avg_frames)


"""empty message

Revision ID: 7b16cf346bda
Revises: 9e8c66caa733
Create Date: 2022-03-28 15:37:47.030500

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '7b16cf346bda'
down_revision = '9e8c66caa733'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('username', sa.String(length=30), nullable=True))
    op.drop_column('user', 'unsername')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('unsername', mysql.VARCHAR(length=30), nullable=True))
    op.drop_column('user', 'username')
    # ### end Alembic commands ###


import requests
import os

# Current working directory
cwd = os.getcwd()

file1 = cwd + r'\input\departments.csv'
file2 = cwd + r'\input\jobs.csv'
file3 = cwd + r'\input\hired_employees.csv'

# Define the URL of your local API
url = 'http://127.0.0.1:5000/upload'

file_paths = [
    (file1, 'departments', ["id", "department"]),
    (file2, 'jobs', ["id", "job"]),
    (file3, 'hired_employees', ["id", "name", "datetime", "department_id", "job_id"])
]

# Create a list of files, associated table names, and additional data
files = [(file_path, table_name, data) for file_path, table_name, data in file_paths]

# Send a single POST request to upload the CSV files with table names and additional data
data = {
    'files': files
}

response = requests.post(url, json=data)

# Print the response
print(response.status_code)
print(response.json())


import argparse
import copy
import abc
import numpy as np
import time
import sys

import openmesh as om
from openmesh import TriMesh

import scipy.sparse as sp
from scipy.sparse import coo_matrix
from scipy.sparse.linalg import eigs

from sksparse.cholmod import cholesky

class Solver:
    """
    Base class of Linear System solver
    """
    def __init__(self, args):
        self.save_file = args.output
        self.input_file = args.input
        self.log = args.verbose

        self.lamba = args.l
        self.kappa = args.k

        self.beta_max = args.beta_max

        self.beta0 = args.beta
        self.beta = self.beta0

        self.input_mesh = om.read_trimesh(self.input_file)

        self.mesh = None
        self._type = None
        self.type = None
        self.t_solver = 'ch'

    def prepare_mesh(self, mesh=None):
        '''
        self.mesh is mesh to be processed
        '''
        if mesh==None:
            self.mesh = copy.deepcopy(self.input_mesh)
        else:
            self.mesh = copy.deepcopy(mesh)

        # set automatic parameters
        if self.lamba == None:
            avl = average_lenth(self.mesh)
            self.lamba = 0.02 * avl**2

    def solve(self):
        print("[ {} ]".format(self._type))
        start_time = time.time()

        print("Processing mesh with ")
        print("vertices: ", self.mesh.n_vertices())
        print("edges: ", self.mesh.n_edges())
        print("faces: ", self.mesh.n_faces())

        self.optimize()

        final_time = time.time()
        self.duration = final_time - start_time
        print("Optimize Time: %f (s)" % (self.optimize_time))
        print("Total Time: %f (s)" % (self.duration))
        print("Iterations: %d" % (self.iteration))

    def save(self, mesh, path=None):
        if path==None:
            om.write_mesh(self.save_file, mesh)
        else:
            om.write_mesh(path, mesh)

    @abc.abstractmethod
    def build_operator(self):
        """
        this paper contain three types of operator:
         - vertex-based cotangent operator
         - cotangent edge operator
         - area-based edge operator

        return: D, p, p*
        """

    def optimize(self) -> TriMesh:
        mesh = self.mesh
        N_edge = mesh.n_edges()
        N_vert = mesh.n_vertices()

        if self.type == 'edge':
            N = N_edge
        else: # type=='vert
            N = N_vert

        D, p, p_star = self.build_operator()
        start_time = time.time()
        # --------------build linear system--------------
        cnt = 1
        while self.beta < self.beta_max:
            # stage 1:
            delta = D @ p

            for i in range(N):
                if delta[i][0]**2 + delta[i][1]**2 + delta[i][2]**2 < self.lamba / self.beta:
                    delta[i][0] = delta[i][1] = delta[i][2] = 0.0

            # stage 2:
            A = sp.eye(N_vert) + self.beta * (D.transpose() @ D)
            b = p_star + self.beta * (D.transpose() @ delta)

            if self.t_solver == 'ch':
                factor = cholesky(A)
            # p, info = sp.linalg.cg(A, b, x0=p)
            for i in range(3):
                if self.t_solver == 'cg':
                    p[:,i], info = sp.linalg.cg(A, b[:,i], x0=p[:,i])
                if self.t_solver == 'ch':
                    # scipy cholesky
                    # y = scipy.linalg.solve_triangular(L, b[:,i], lower=True, check_finite=False)
                    # p[:,i] = scipy.linalg.solve_triangular(L.T, y, lower=True, check_finite=False)
                    p[:,i] = factor.solve_A(b[:,i])
            if self.log:
                print('iter {}: beta={}, p[0,0]={}'.format(cnt, self.beta, p[0][0]))
            cnt += 1
            self.beta *= self.kappa
        self.denoised_vert = p
        final_time = time.time()
        self.optimize_time = final_time - start_time
        self.iteration = cnt
        # --------------return mesh--------------
        for i, vh in enumerate(mesh.vertices()):
            mesh.set_point(vh, p[i])
        self.denoised_mesh = mesh
        return mesh

#---------------------utils--------------------------#

def average_lenth(mesh) -> float:
        lenth = 0
        for eh in mesh.edges():
            lenth += mesh.calc_edge_length(eh)
        return lenth / mesh.n_edges()

def add_noise(mesh, l=0.3, set_seed=777):
    """
    return a new mesh with noise of l * average edge length
    """
    np.random.seed(set_seed)
    input_mesh_avglen = average_lenth(mesh)
    ret = copy.deepcopy(mesh)
    for vh in ret.vertices():
        point = ret.point(vh)
        noise = np.random.normal(0, input_mesh_avglen * l)
        new_point = point + noise
        ret.set_point(vh, new_point)
    return ret

#---------------------Methods--------------------------#

class VertexSolver(Solver):
    def __init__(self, args):
        super(VertexSolver, self).__init__(args)
        self._type = 'Vertex Based Method'
        self.type = 'vert'

    def build_operator(self):
        mesh = self.mesh
        N_edge = mesh.n_edges()
        N_vert = mesh.n_vertices()

        # --------------build D--------------
        D_row = []
        D_col = []
        D_data = []

        for cnt, vh in enumerate(mesh.vertices()):
            neighbor_vhs = [vh for vh in mesh.vv(vh)]
            cots = []
            for neighbor_vh in neighbor_vhs:
                heh = mesh.find_halfedge(vh, neighbor_vh)
                vh1 = mesh.to_vertex_handle(heh)
                vh3 = mesh.from_vertex_handle(heh)
                vh2 = mesh.opposite_vh(heh)
                vh4 = mesh.opposite_he_opposite_vh(heh)

                p1, p2, p3, p4 = mesh.point(vh1), mesh.point(vh2), mesh.point(vh3), mesh.point(vh4)

                cot123 = np.dot(p1-p2, p3-p2)/np.linalg.norm(np.cross(p1-p2, p3-p2))
                cot341 = np.dot(p3-p4, p1-p4)/np.linalg.norm(np.cross(p3-p4, p1-p4))

                cots.append(cot123+cot341)
            cot_sum = sum(cots)
            for i, neighbor_vh in enumerate(neighbor_vhs):
                D_row.append(cnt)
                D_col.append(neighbor_vh.idx())
                D_data.append(cots[i]/cot_sum)
            D_row.append(cnt)
            D_col.append(vh.idx())
            D_data.append(-1)

        D = coo_matrix((D_data, (D_row, D_col)), shape=(N_vert, N_vert))
        p = np.array([mesh.point(vertex) for vertex in mesh.vertices()])
        p_star = copy.deepcopy(p)
        return D, p, p_star

class VertexSolver_FFT(VertexSolver):
    def __init__(self, args):
        super(VertexSolver_FFT, self).__init__(args)
        self._type = 'Vertex Based Method (Using FFT)'

    def decomposite(self, L):
        print('computing eigen decomposition')
        num_eigen = 100
        eignvalue, V = eigs(L, k=num_eigen)
        E = np.diag(eignvalue)
        print("done decomposition")
        return V, E, V.transpose()

    def optimize(self) -> TriMesh:
        mesh = self.mesh
        N_edge = mesh.n_edges()
        N_vert = mesh.n_vertices()

        if self.type == 'edge':
            N = N_edge
        else: # type=='vert
            N = N_vert

        D, p, p_star = self.build_operator()
        U, _, UT = self.decomposite(D)

        MTF = np.power(np.linalg.norm(UT@(D.transpose()@D), ord='fro'), 2)
        start_time = time.time()
        # --------------build linear system--------------
        cnt = 1
        while self.beta < self.beta_max:
            # stage 1:
            delta = D @ p

            for i in range(N):
                if delta[i][0]**2 + delta[i][1]**2 + delta[i][2]**2 < self.lamba / self.beta:
                    delta[i][0] = delta[i][1] = delta[i][2] = 0.0

            # stage 2:
            div = 1 + self.beta * MTF
            for i in range(3):
                res = UT@p_star[:,i] + self.beta * UT @ (D.transpose() @ delta[:,i])
                res = res / div
                res = U @ res
                p[:,i] = res

            if self.log:
                print('iter {}: beta={}, p[0,0]={}'.format(cnt, self.beta, p[0][0]))
            cnt += 1
            self.beta *= self.kappa
        self.denoised_vert = p
        final_time = time.time()
        self.optimize_time = final_time - start_time
        self.iteration = cnt
        # --------------return mesh--------------
        for i, vh in enumerate(mesh.vertices()):
            mesh.set_point(vh, p[i])
        self.denoised_mesh = mesh
        return mesh

class CotEdgeSolver(Solver):
    def __init__(self, args):
        super(CotEdgeSolver, self).__init__(args)
        self._type = 'Edge Based Method'
        self.type = 'edge'

    def build_operator(self):
        mesh = self.mesh
        N_edge = mesh.n_edges()
        N_vert = mesh.n_vertices()

        # --------------build D--------------
        D_row = []
        D_col = []
        D_data = []

        for cnt, eh in enumerate(mesh.edges()):
            heh1 = mesh.next_halfedge_handle(mesh.halfedge_handle(eh, 0)) # next edge of 1st halfedge of e
            heh2 = mesh.next_halfedge_handle(mesh.halfedge_handle(eh, 1)) # next edge of 2nd halfedge of e

            vh3, vh4 = mesh.from_vertex_handle(heh1), mesh.to_vertex_handle(heh1)
            vh1, vh2 = mesh.from_vertex_handle(heh2), mesh.to_vertex_handle(heh2)
            # corresponding to Figure 3
            p1, p2, p3, p4 = mesh.point(vh1), mesh.point(vh2), mesh.point(vh3), mesh.point(vh4)

            S123 = 0.5 * np.linalg.norm(np.cross(p2 - p1, p2 - p3))
            S134 = 0.5 * np.linalg.norm(np.cross(p4 - p1, p4 - p3))
            l13 = np.linalg.norm(p1 - p3)

            cot231 = np.dot(p2-p3, p1-p3)/np.linalg.norm(np.cross(p2-p3, p1-p3))
            cot134 = np.dot(p1-p3, p4-p3)/np.linalg.norm(np.cross(p1-p3, p4-p3))
            cot312 = np.dot(p3-p1, p2-p1)/np.linalg.norm(np.cross(p3-p1, p2-p1))
            cot413 = np.dot(p4-p1, p3-p1)/np.linalg.norm(np.cross(p4-p1, p3-p1))

            coef1 = -1 * cot231 - cot134
            coef2 = cot231 + cot312
            coef3 = -1 * cot312 - cot413
            coef4 = cot134 + cot413

            D_row.append(cnt)
            D_col.append(vh1.idx())
            D_data.append(coef1)

            D_row.append(cnt)
            D_col.append(vh2.idx())
            D_data.append(coef2)

            D_row.append(cnt)
            D_col.append(vh3.idx())
            D_data.append(coef3)

            D_row.append(cnt)
            D_col.append(vh4.idx())
            D_data.append(coef4)

        D = coo_matrix((D_data, (D_row, D_col)), shape=(N_edge, N_vert))
        p = np.array([mesh.point(vertex) for vertex in mesh.vertices()])
        p_star = copy.deepcopy(p)
        return D, p, p_star

class AreaEdgeSolver(Solver):
    def __init__(self, args):
        super(AreaEdgeSolver, self).__init__(args)
        self._type = 'Area Based Method'
        self.type = 'edge'

    def build_operator(self):
        mesh = self.mesh
        N_edge = mesh.n_edges()
        N_vert = mesh.n_vertices()

        # --------------build D--------------
        D_row = []
        D_col = []
        D_data = []

        for cnt, eh in enumerate(mesh.edges()):
            heh1 = mesh.next_halfedge_handle(mesh.halfedge_handle(eh, 0)) # next edge of 1st halfedge of e
            heh2 = mesh.next_halfedge_handle(mesh.halfedge_handle(eh, 1)) # next edge of 2nd halfedge of e

            vh3, vh4 = mesh.from_vertex_handle(heh1), mesh.to_vertex_handle(heh1)
            vh1, vh2 = mesh.from_vertex_handle(heh2), mesh.to_vertex_handle(heh2)
            # corresponding to Figure 3
            p1, p2, p3, p4 = mesh.point(vh1), mesh.point(vh2), mesh.point(vh3), mesh.point(vh4)

            S123 = 0.5 * np.linalg.norm(np.cross(p2 - p1, p2 - p3))
            S134 = 0.5 * np.linalg.norm(np.cross(p4 - p1, p4 - p3))
            l13 = np.linalg.norm(p1 - p3)

            coef1 = (S123 * np.dot(p4 - p3, p3 - p1) + S134 * np.dot(p1 - p3, p3 - p2)) / (l13 * l13 * (S123 + S134))
            coef2 = S134 / (S123 + S134)
            coef3 = (S123 * np.dot(p3 - p1, p1 - p4) + S134 * np.dot(p2 - p1, p1 - p3)) / (l13 * l13 * (S123 + S134))
            coef4 = S123 / (S123 + S134)

            D_row.append(cnt)
            D_col.append(vh1.idx())
            D_data.append(coef1)

            D_row.append(cnt)
            D_col.append(vh2.idx())
            D_data.append(coef2)

            D_row.append(cnt)
            D_col.append(vh3.idx())
            D_data.append(coef3)

            D_row.append(cnt)
            D_col.append(vh4.idx())
            D_data.append(coef4)

        D = coo_matrix((D_data, (D_row, D_col)), shape=(N_edge, N_vert))
        p = np.array([mesh.point(vertex) for vertex in mesh.vertices()])
        p_star = copy.deepcopy(p)
        return D, p, p_star

class AreaEdgeSolver_R(Solver):
    def __init__(self, args):
        super(AreaEdgeSolver_R, self).__init__(args)
        self._type = 'Area Based Method w/ reg'
        self.type = 'edge'
        self.alpha = args.a

    def get_gemma(self, mesh):
        PI = 3.14159265359
        cnt, sum = 0, 0
        for heh in mesh.halfedges():
            angle = mesh.calc_dihedral_angle(heh) * (180/PI)
            sum += abs(angle)
            cnt += 1
        return sum/cnt

    def build_operator(self):
        mesh = self.mesh
        N_edge = mesh.n_edges()
        N_vert = mesh.n_vertices()

        if self.alpha == 0:
            self.alpha = 0.1 * self.get_gemma(mesh)

        # --------------build D and R--------------
        D_row = []
        D_col = []
        D_data = []

        R_data = []

        for cnt, eh in enumerate(mesh.edges()):
            heh1 = mesh.next_halfedge_handle(mesh.halfedge_handle(eh, 0)) # next edge of 1st halfedge of e
            heh2 = mesh.next_halfedge_handle(mesh.halfedge_handle(eh, 1)) # next edge of 2nd halfedge of e

            vh3, vh4 = mesh.from_vertex_handle(heh1), mesh.to_vertex_handle(heh1)
            vh1, vh2 = mesh.from_vertex_handle(heh2), mesh.to_vertex_handle(heh2)
            # corresponding to Figure 3
            p1, p2, p3, p4 = mesh.point(vh1), mesh.point(vh2), mesh.point(vh3), mesh.point(vh4)

            S123 = 0.5 * np.linalg.norm(np.cross(p2 - p1, p2 - p3))
            S134 = 0.5 * np.linalg.norm(np.cross(p4 - p1, p4 - p3))
            l13 = np.linalg.norm(p1 - p3)

            coef1 = (S123 * np.dot(p4 - p3, p3 - p1) + S134 * np.dot(p1 - p3, p3 - p2)) / (l13 * l13 * (S123 + S134))
            coef2 = S134 / (S123 + S134)
            coef3 = (S123 * np.dot(p3 - p1, p1 - p4) + S134 * np.dot(p2 - p1, p1 - p3)) / (l13 * l13 * (S123 + S134))
            coef4 = S123 / (S123 + S134)

            D_row.append(cnt)
            D_col.append(vh1.idx())
            D_data.append(coef1)

            D_row.append(cnt)
            D_col.append(vh2.idx())
            D_data.append(coef2)

            D_row.append(cnt)
            D_col.append(vh3.idx())
            D_data.append(coef3)

            D_row.append(cnt)
            D_col.append(vh4.idx())
            D_data.append(coef4)

            R_data.append(1)
            R_data.append(-1)
            R_data.append(1)
            R_data.append(-1)

        D = coo_matrix((D_data, (D_row, D_col)), shape=(N_edge, N_vert))
        R = coo_matrix((R_data, (D_row, D_col)), shape=(N_edge, N_vert))
        p = np.array([mesh.point(vertex) for vertex in mesh.vertices()])
        p_star = copy.deepcopy(p)
        return D, R, p, p_star

    def optimize(self) -> TriMesh:
        mesh = self.mesh
        N_edge = mesh.n_edges()
        N_vert = mesh.n_vertices()

        if self.type == 'edge':
            N = N_edge
        else: # type=='vert
            N = N_vert

        D, R, p, p_star = self.build_operator()
        start_time = time.time()
        # --------------build linear system--------------
        cnt = 1
        while self.beta < self.beta_max:
            # stage 1:
            delta = D @ p

            for i in range(N):
                if delta[i][0]**2 + delta[i][1]**2 + delta[i][2]**2 < self.lamba / self.beta:
                    delta[i][0] = delta[i][1] = delta[i][2] = 0.0

            # stage 2:
            A = sp.eye(N_vert) + self.alpha * (R.transpose() @ R) + self.beta * (D.transpose() @ D)
            b = p_star + self.beta * (D.transpose() @ delta)

            if self.t_solver == 'ch':
                factor = cholesky(A)
            # p, info = sp.linalg.cg(A, b, x0=p)
            for i in range(3):
                if self.t_solver == 'cg':
                    p[:,i], info = sp.linalg.cg(A, b[:,i], x0=p[:,i])
                if self.t_solver == 'ch':
                    # scipy cholesky
                    # y = scipy.linalg.solve_triangular(L, b[:,i], lower=True, check_finite=False)
                    # p[:,i] = scipy.linalg.solve_triangular(L.T, y, lower=True, check_finite=False)
                    p[:,i] = factor.solve_A(b[:,i])
            if self.log:
                print('iter {}: beta={}, p[0,0]={}'.format(cnt, self.beta, p[0][0]))
            cnt += 1
            self.beta *= self.kappa
            self.alpha *= 0.5
        self.denoised_vert = p
        final_time = time.time()
        self.optimize_time = final_time - start_time
        self.iteration = cnt
        # --------------return mesh--------------
        for i, vh in enumerate(mesh.vertices()):
            mesh.set_point(vh, p[i])
        self.denoised_mesh = mesh
        return mesh

if __name__=='__main__':
    parser = argparse.ArgumentParser(description="implementation of image smoothing via L0 gradient minimization")

    #parser.add_argument('--input', default='./input/fandisk.obj',
    parser.add_argument('--input', default='./input/SharpSphere.obj',
        help="input mesh file")

    parser.add_argument('--output', default='./res.obj',
        help="output mesh file")

    parser.add_argument('-k', type=float, default=1.414,
        metavar='kappa', help='updating weight (default 1.414), refer mu in mesh2013')

    parser.add_argument('-l', type=float, default=1./16,
        metavar='lambda', help='smoothing weight (None is automatic)')

    parser.add_argument('--beta', type=float, default=1.0e-3,
        help='beta init value (default 1.0e-3)')

    parser.add_argument('--beta_max', type=float, default=1e3,
        help='updating threshold (default 1e3)')

    parser.add_argument('-v', '--verbose', action='store_true',
        help='enable verbose logging for each iteration')

    parser.add_argument('-m', default='edge',
        metavar='method', help="support three method: vert, edge, area")

    parser.add_argument('-n', type=float, default=0.3,
        metavar='noise', help="ratio of average edge length")

    parser.add_argument('-r', '--regulation', action='store_true',
        help='use regulation')

    parser.add_argument('-a', type=float, default=0,
        metavar='alpha', help="parameter of regulation")

    args = parser.parse_args()

    #s = LinearSystem_Solver(args)
    if args.m == 'vert':
        s = VertexSolver(args)
    elif args.m == 'edge':
        s = CotEdgeSolver(args)
    elif args.m == 'area':
        if args.regulation:
            s = AreaEdgeSolver_R(args)
        else:
            s = AreaEdgeSolver(args)
    elif args.m == 'vfft':
        s = VertexSolver_FFT(args)

    noise_mesh = add_noise(s.input_mesh, l=float(args.n))
    s.save(noise_mesh, './noise.obj')
    s.prepare_mesh(mesh=noise_mesh)
    s.solve()
    s.save(s.denoised_mesh)

    print('done!')


from bs4 import BeautifulSoup
import requests
from util import init_logger, Logger, log_args
import logging
import argparse
import xml.etree.ElementTree as ET
import config
import time
import random
import pickle as pkl

def _build_parser():
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('--log_level',
                        help=str([value for key, value in logging._levelToName.items()]) + " ( default : INFO )",
                        default=logging._levelToName[logging.INFO])
    # parser.add_argument('--job_type', help=str([value for value in JOB_TYPE_LIST]), required=True)

    return parser

def _indent(elem, level=1):
    i = "\n" + level * "  "
    if len(elem):
        if not elem.text or not elem.text.strip():
            elem.text = i + "  "
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
        for elem in elem:
            _indent(elem, level + 1)
        if not elem.tail or not elem.tail.strip():
            elem.tail = i
    else:
        if level and (not elem.tail or not elem.tail.strip()):
            elem.tail = i

'''
    h2 tag + class = jobtitle
'''

def _get_html(url):
    _html = ""
    resp = requests.get(url)
    if resp.status_code == 200:
        _html = resp.text
    return _html

'''
    각 페이지에서 채용 공고의 Key들을 불러옴
'''

def get_emp_key(url,index):
    # 해당 페이지에 "다음" 글씨가 있는지 여부
    # next_page_exist=True

    req_url = url +"&start=" + str(index * 50)
    Logger.debug("get_emp_key request URL = " + url)
    html = _get_html(url)

    soup = BeautifulSoup(html, 'html.parser')

    job_key_list = []

    # a_tag 중 채용공고에 대한 Tag를 찾는 방식
    # -> 일반 채용공고 뿐만 아니라 유료 광고도 불러옴 (일반 채용 공고 10개 + 유료 광고 6개)
    for a_tag in soup.find_all("a", {"data-tn-element": "jobTitle"}):
        split_id = a_tag.parent["id"].split("_")

        id_type = split_id[0]
        emp_id = split_id[1]

        job_key_list.append(emp_id)

    return job_key_list

'''
    API를 이용하여 job key 가져오기
'''

def get_emp_key_from_api(location, index):
    # api는 최대 25개 까지 1번에 읽기 가능...
    url = config.INDEED_API_SEARCH_URL + "&l=" + location + "&start=" + str(index * 25)

'''
    채용 공고 내용 불러오기

    # Return
        xml : 채용 공고 내용을 xml 형태로 구성

'''

def get_job_detail(key):
    job_dict = {}
    req_url = config.INDEED_JOB_SEARCH_URL + key
    html = _get_html(req_url)
    soup = BeautifulSoup(html, 'html.parser')

    # job header 내용 Parsing
    job_header = soup.find("div", {"data-tn-component": "jobHeader"})
    Logger.debug("request URL =" + req_url)
    # print(job_header)

    job_title = job_header.find("b", {"class": "jobtitle"}).contents[0].get_text()
    company = job_header.find("span", {"class": "company"}).get_text()
    location = job_header.find("span", {"class": "location"}).get_text()
    cmp_description = ""

    soup.find("span", {"class": "company"})
    # company description이 없는 페이지도 있음
    try:
        cmp_description = soup.find("div", {"class": "cmp_description"}).get_text()
    except AttributeError:
        Logger.debug("cmp_description does not exists")

    job_summary = soup.find("span", {"id": "job_summary"}).get_text()

    Logger.debug("jobtitle = " + job_title)
    Logger.debug("company = " + company)
    Logger.debug("location = " + location)
    Logger.debug("cmp_description = " + cmp_description)
    Logger.debug("job_summary = " + job_summary)

    xml = ET.Element("root")
    ET.SubElement(xml, "job_key").text = key
    ET.SubElement(xml, "jobtitle").text = job_title
    ET.SubElement(xml, "company").text = company
    ET.SubElement(xml, "location").text = location
    ET.SubElement(xml, "cmp_description").text = cmp_description
    ET.SubElement(xml, "job_summary").text = job_summary

    Logger.debug(ET.tostring(xml, encoding="utf=8"))
    _indent(xml)

    return xml

def _write_xml(xml, file_path):
    ET.ElementTree(xml).write(file_path, encoding="UTF-8")

# 0.1~sec초 간격으로 randomly sleep
def rand_sleep(sec):
    sleep_len = random.uniform(0.1, sec)
    Logger.debug("sleep " + str(sleep_len) + "sec")
    time.sleep(sleep_len)

'''
    url_list에 해당하는
    채용 공고를 읽어와서 xml로 씀
'''
def write_emp_detail(url_list,max_sleep):

    crawling_data_num = 0
    start_time = time.time()

    # job detail을 불러와 xml파일로 쓴 job key들
    key_dict = {}

    # 요청시 실패한 key list 저장
    failed_key_list = []
    url_index=0
    # URL 당 최대 1000개의 job detail적음
    for url in url_list:
        Logger.info(str(url_index+1) + "/"+str(len(url_list))+" job list request URL = "+url.replace("\n",""))
        url_index+=1
        job_key_list=[]

        # 채용 공고 목록 페이지에서 job key들을 받아옴
        # 50개씩 최대 20번 불러옴
        try:
            req_url = config.INDEED_URL+url+"&limit=50"
            prev_key_list=[]
            for index in range(20):
                cur_key_list = get_emp_key(req_url,index)
                # 페이지 index가 넘어간 경우 계속 똑같은 list를 반복하므로 이 경우 더 이상 job key를 받아오지 않음
                if prev_key_list==cur_key_list:
                    break
                else:
                    prev_key_list=cur_key_list
                job_key_list=job_key_list+cur_key_list
                rand_sleep(max_sleep)
        except:
            continue

        Logger.info("job key list length = "+str(len(job_key_list)))

        not_exist_cnt = 0

        # job key들의 채용 공고 detail 받아옴
        for key in job_key_list:
            # 해당 key에 대해서 아직 Crwaling 안한 경우
            # Crawling 수행하고 xml 파일 적음
            if not key in key_dict:
                key_dict[key] = True
                try:
                    xml = get_job_detail(key)
                except:
                    Logger.warn(key + " parsing failed")
                    failed_key_list.append(key)
                    rand_sleep(max_sleep)
                    continue

                file_path = config.JOB_DETAIL_FILE_PATH + key + ".xml"
                _write_xml(xml, file_path)
                Logger.debug(file_path+" is written")
                not_exist_cnt += 1
                crawling_data_num += 1
                rand_sleep(max_sleep)
            else:
                Logger.debug("key " + key + " already exist")

        # url
        Logger.info("total data num =" + str(crawling_data_num) + " added data num =" + str(
                            not_exist_cnt) + " elapsed time = " + str(time.time() - start_time)+"\n")

        # 현재까지 가져온 모든 key들을 pkl로 저장
        key_dict_path = "key_dict.pkl"
        # 적은 job key의 dictionary를 pickle 파일로 저장
        with open(key_dict_path, "wb") as f:
            pkl.dump(key_dict, f)

        Logger.info(key_dict_path + "has been written")

                        # if not_exist_cnt==0:
                        #     Logger.info("threre are not any employment not written")
                        #     break

# if __name__ == "__main__":
#
#     parser = _build_parser()
#     FLAGS = parser.parse_args()
#     init_logger(logging._nameToLevel[FLAGS.log_level])
#     log_args(FLAGS)
#
#     job_type = FLAGS.job_type
#
#     key_dict = {}
#
#     # 요청시 실패한 key list 저장
#     failed_key_list = []
#
#     index = 0
#     crawling_data_num = 0
#
#     start_time = time.time()
#
#     while True:
#
#         Logger.info(str(index) + "th job list request")
#         job_key_list = get_emp_key(job_type, 25)
#         # job_key_list = get_emp_key(job_type,index)
#
#         rand_sleep(MAX_SLEEP_SEC)
#         index += 1
#
#         not_exist_cnt = 0
#         for key in job_key_list:
#             # 해당 key에 대해서 아직 Crwaling 안한 경우
#             # Crawling 수행
#             if not key in key_dict:
#                 key_dict[key] = True
#                 try:
#                     xml = get_job_detail(key)
#                 except:
#                     Logger.warn(key + " parsing failed")
#                     failed_key_list.append(key)
#                     rand_sleep(MAX_SLEEP_SEC)
#                     continue
#
#                 file_path = XML_FILE_PATH + key + ".xml"
#                 _write_xml(xml, file_path)
#                 not_exist_cnt += 1
#                 crawling_data_num += 1
#                 rand_sleep(MAX_SLEEP_SEC)
#
#         Logger.info("total data num =" + str(crawling_data_num) + " added data num =" + str(
#             not_exist_cnt) + " elapsed time = " + str(time.time() - start_time))
#
#         # if not_exist_cnt==0:
#         #     Logger.info("threre are not any employment not written")
#         #     break
#
#     # 실패한 Key들 적음
#     with open("failed_key_list.txt", "w") as f:
#         f.write("\n".join(failed_key_list))
#
#

        # html = _get_html(INDEED_QUERY_URL)
        # soup = BeautifulSoup(html, 'html.parser')
        # result = soup.find_all("h2")
        # result = soup.h2['job']
        # # print(soup)
        # print(result)