from __future__ import absolute_import
from __future__ import print_function
from .._abstract.abstract import BaseAGSServer
import json
########################################################################
class System(BaseAGSServer):
    """
    The System resource is a collection of miscellaneous server-wide
    resources such as server properties, server directories, the
    configuration store, Web Adaptors, and licenses.
    """
    _json = None
    _json_dict = None
    _proxy_url = None
    _proxy_port = None
    _securityHandler = None
    _url = None
    _resources = None
    #----------------------------------------------------------------------
    def __init__(self, url, securityHandler,
                 proxy_url=None,
                 proxy_port=None,
                 initialize=False):
        """Constructor"""
        self._proxy_url = proxy_url
        self._proxy_port = proxy_port
        self._securityHandler = securityHandler
        if url.lower().endswith("/system"):
            self._url = url
        else:
            self._url = url + "/system"
        if initialize:
            self.__init()
    #----------------------------------------------------------------------
    def __init(self):
        """ populates server admin information """
        params = {
            "f" : "json"
        }
        json_dict = self._get(url=self._url, param_dict=params,
                                 securityHandler=self._securityHandler,
                                 proxy_url=self._proxy_url,
                                 proxy_port=self._proxy_port)
        self._json = json.dumps(json_dict)
        self._json_dict = json_dict
        attributes = [attr for attr in dir(self)
                      if not attr.startswith('__') and \
                      not attr.startswith('_')]
        for k,v in json_dict.items():
            if k in attributes:
                setattr(self, "_"+ k, json_dict[k])
            else:
                print( k, " - attribute not implemented in System.")
            del k
            del v
    #----------------------------------------------------------------------
    def __str__(self):
        """returns the object as a string"""
        return json.dumps(self._json)
    #----------------------------------------------------------------------
    @property
    def resources(self):
        """gets the resources"""
        if self._resources is None:
            self.__init()
        return self._resources
    #----------------------------------------------------------------------
    @property
    def serverProperties(self):
        """gets the server properties for the site as an object"""
        return ServerProperties(url=self._url + "/properties",
                                securityHandler=self._securityHandler,
                                proxy_url=self._proxy_url,
                               proxy_port=self._proxy_port,
                               initialize=True)
    #----------------------------------------------------------------------
    @property
    def serverDirectories(self):
        """returns the server directory object in a list"""
        directs = []
        url = self._url + "/directories"
        params = {
            "f" : "json"
        }
        res = self._get(url=url,
                           param_dict=params,
                           securityHandler=self._securityHandler,
                           proxy_url=self._proxy_url,
                           proxy_port=self._proxy_port)
        for direct in res['directories']:
            directs.append(
                ServerDirectory(url=url + "/%s" % direct["name"],
                                securityHandler=self._securityHandler,
                                proxy_url=self._proxy_url,
                                proxy_port=self._proxy_port,
                                initialize=True))
        return directs
    #----------------------------------------------------------------------
    @property
    def directories(self):
        """returns the dictionary with all the server directories properties"""
        url = self._url + "/directories"
        params = {
            "f" : "json"
        }
        res = self._get(url=url,
                        param_dict=params,
                        securityHandler=self._securityHandler,
                        proxy_url=self._proxy_url,
                        proxy_port=self._proxy_port)

        return res
    #----------------------------------------------------------------------
    def registerDirectory(self,name,physicalPath,directoryType,cleanupMode,
                          maxFileAge,description):
        """
        Registers a new server directory. While registering the server directory,
        you can also specify the directory's cleanup parameters. You can also
        register a directory by using its JSON representation as a value of the
        directory parameter.
        Inputs:
            name - The name of the server directory.
            physicalPath - The absolute physical path of the server directory.
            directoryType - The type of server directory.
            cleanupMode - Defines if files in the server directory needs to
                            be cleaned up. Default: NONE
            maxFileAge - Defines how long a file in the directory needs to be
                            kept before it is deleted (in minutes).
            description - An optional description for the server directory.
        """
        url = self._url + "/directories/register"
        params = {
            "f" : "json",
            "name" : name,
            "physicalPath" : physicalPath,
            "directoryType" : directoryType,
            "cleanupMode" : cleanupMode,
            "maxFileAge" : maxFileAge,
            "description" : description
        }

        res = self._post(url=url,
                         param_dict=params,
                         securityHandler=self._securityHandler,
                         proxy_url=self._proxy_url,
                         proxy_port=self._proxy_port)

        return res
    #----------------------------------------------------------------------
    def registerDirs(self,json_dirs):
        """
        Registers multiple new server directories.
        Inputs:
            json_dirs - Array of Server Directories in JSON format.
        """
        url = self._url + "/directories/registerDirs"
        params = {
            "f" : "json",
            "directories" : json_dirs
        }

        res = self._post(url=url,
                         param_dict=params,
                         securityHandler=self._securityHandler,
                         proxy_url=self._proxy_url,
                         proxy_port=self._proxy_port)

        return res
    #----------------------------------------------------------------------
    def recover(self,runAsync=False):
        """
        If the shared server directories for a site are unavailable, a site
        in read-only mode will operate in a degraded capacity that allows access
        to the ArcGIS Server Administrator Directory. You can recover a site if
        the shared server directories are permanently lost. The site must be in
        read-only mode, and the site configuration files must have been copied
        to the local repository when switching site modes. The recover operation
        will copy the server directories from the local repository into the
        shared server directories location. The copied local repository will be
        from the machine in the site where the recover operation is performed.
        Inputs:
           runAsync - default False - Decides if this operation must run
            asynchronously.
        """
        url = self._url + "/directories/recover"
        params = {
            "f" : "json",
            "runAsync" : runAsync
        }

        res = self._get(url=url,
                        param_dict=params,
                        securityHandler=self._securityHandler,
                        proxy_url=self._proxy_url,
                        proxy_port=self._proxy_port)

        return res
    #----------------------------------------------------------------------
    @property
    def licenses(self):
        """
        The licenses resource lists the current license level of ArcGIS for
        Server and all authorized extensions. Contact Esri Customer Service
        if you have questions about license levels or expiration properties.
        """
        url = self._url + "/licenses"
        params = {
            "f" : "json"
        }

        return self._get(url=url,
                            param_dict=params,
                            securityHandler=self._securityHandler,
                            proxy_url=self._proxy_url,
                            proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    @property
    def Jobs(self):
        """get the Jobs object"""
        url = self._url + "/jobs"
        return Jobs(url=url,
                    securityHandler=self._securityHandler,
                    proxy_url=self._proxy_url,
                    proxy_port=self._proxy_port,
                    initialize=True)
    #----------------------------------------------------------------------
    @property
    def webAdaptors(self):
        """
        This operation lists all the Web Adaptors that have been registered
        with the site. The server will trust all these Web Adaptors and
        will authorize calls from these servers.
        To configure a new Web Adaptor with the server, you'll need to use
        the configuration web page or the command line utility. For full
        instructions, see Configuring the Web Adaptor after installation.
        """
        url = self._url + "/webadaptors"
        params = {
            "f" : "json"
        }
        return self._get(url=url, param_dict=params,
                            securityHandler=self._securityHandler,
                            proxy_url=self._proxy_url,
                            proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    @property
    def webAdaptorsConfiguration(self):
        """
        The Web Adaptors configuration is a resource for all the
        configuration parameters shared across all the Web Adaptors in the
        site. Most importantly, this resource lists the shared key that is
        used by all the Web Adaptors to encrypt key data bits in the
        incoming requests to the server.
        """
        url = self._url + "/webadaptors/config"
        params = {
            "f" : "json"
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    def updateWebAdaptorsConfiguration(self, webAdaptorConfig):
        """
        You can use this operation to change the configuration parameters
        and shared key.

        Inputs:
           webAdaptorConfig - the sharedKey attribute must always be
            present in this JSON
        """
        url = self._url + "/webadaptors/config/update"
        params = {
            "f" : "json",
            "webAdaptorConfig" : webAdaptorConfig
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    def registerWebAdaptor(self, webAdaptorURL, machineName, machineIP,
                           isAdminEnabled, description, httpPort, httpsPort):
        """
        You can use this operation to register the ArcGIS Web Adaptor
        from your ArcGIS Server. By registering the Web Adaptor with the server,
        you are telling the server to trust requests (including security
        credentials) that have been submitted through this Web Adaptor.

        Inputs:
           webAdaptorURL - The URL of the web adaptor through which ArcGIS
                            resources will be accessed.
           machineName - The machine name on which the web adaptor is installed.
           machineIP - The local IP address of the machine on which the web
                            adaptor is installed.
           isAdminEnabled - A boolean flag to indicate if administrative access
                            is allowed through the web adaptor. The default is
                            false.
           description - An optional description for the web adaptor.
           httpPort - An optional parameter to indicate the HTTP port of the
                            web adaptor. If this parameter is not provided, it
                            is derived from the URL.
           httpsPort - An optional parameter to indicate the HTTPS port of the web
                            adaptor. If this parameter is not provided, it is
                            derived from the URL.
        """
        url = self._url + "/webadaptors/register"
        params = {
            "f" : "json",
            "webAdaptorURL" : webAdaptorURL,
            "machineName" : machineName,
            "machineIP" : machineIP,
            "isAdminEnabled" : isAdminEnabled,
            "description" : description,
            "httpPort" : httpPort,
            "httpsPort" :  httpsPort
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_port=self._proxy_port,
                             proxy_url=self._proxy_url)
    #----------------------------------------------------------------------
    def unregisterWebAdaptor(self, webAdaptorID):
        """
        You can use this operation to unregister the ArcGIS Web Adaptor
        from your ArcGIS Server. Once a Web Adaptor has been unregistered,
        your ArcGIS Server will no longer trust the Web Adaptor and will not
        accept any credentials from it. This operation is typically used when
        you want to register a new Web Adaptor or when your old Web Adaptor
        needs to be updated. After unregistering, the Web Adaptor can no
        longer submit requests to the server.

        Inputs:
           webAdaptorID - id of the web adaptor
        """
        url = self._url + "/webadaptors/%s/unregister" % webAdaptorID
        params = {
            "f" : "json"
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_port=self._proxy_port,
                             proxy_url=self._proxy_url)
    #----------------------------------------------------------------------
    @property
    def configurationStore(self):
        """returns the ConfigurationStore object for this site"""
        url = self._url + "/configstore"

        return ConfigurationStore(url=url,
                                  securityHandler=self._securityHandler,
                                  proxy_url=self._proxy_url,
                                  proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    def clearRestCache(self):
        """
        This operation clears the cache on all REST handlers in the system.
        While the server typically manages the REST cache for you, use this
        operation to get explicit control over the cache.
        Note: Invoking this operation will cause REST handlers to be
        restarted on all server machines.
        """
        url = self._url + "/handlers/rest/cache/clear"
        params = {
            "f" : "json"
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_port=self._proxy_port,
                             proxy_url=self._proxy_url)
    @property
    #----------------------------------------------------------------------
    def servicesDirectory(self):
        """
        This resource lists properties related to the HTML view of the ArcGIS
        REST API (also known as the Services Directory). You can edit these
        properties if you want to disable the Services Directory or point its
        previews at a locally-hosted JavaScript API or map viewer.
        Returns a dictionary with the properties related to Services Directory.
        """
        url = self._url + "/handlers/rest/servicesdirectory"
        params = {
            "f" : "json"
        }
        return self._get(url=url,
                        param_dict=params,
                        securityHandler=self._securityHandler,
                        proxy_port=self._proxy_port,
                        proxy_url=self._proxy_url)
    #----------------------------------------------------------------------
    def editServicesDirectory(self,params):
        """
        With this operation you can enable or disable the HTML view of ArcGIS
        REST API (also known as the Services Directory). You can also adjust
        the JavaScript and map viewer previews of services in the Services
        Directory so that they work with your own locally hosted JavaScript
        API and map viewer.
        Note: When the Services Directory is disabled, the HTML view is not
        accessible. However, the REST API is still available.

        Inputs:
            allowedOrigins - Comma-separated list of URLs of domains allowed
                            to make requests. * can be used to denote all
                            domains.
            arcgis.com.map - URL of the map viewer application used for service
                            previews. Defaults to the ArcGIS.com map viewer but
                            could be used to point at your own Portal for
                            ArcGIS map viewer.
            arcgis.com.map.text - The text to use for the preview link that
                            opens the map viewer.
            jsapi.arcgis - The URL of the JavaScript API to use for service
                            previews. Defaults to the online ArcGIS API for
                            JavaScript, but could be pointed at your own
                            locally-installed instance of the JavaScript API.
            jsapi.arcgis.css - CSS file associated with the ArcGIS API for
                            JavaScript. Defaults to the online Dojo tundra.css.
            jsapi.arcgis.css2 - Additional CSS file associated with the ArcGIS
                            API for JavaScript. Defaults to the online esri.css.
            jsapi.arcgis.sdk - URL of the ArcGIS API for JavaScript help.
            servicesDirEnabled - Flag to enable/disable the HTML view of the
                            services directory. Values: true | false
        """
        url = self._url + "/handlers/rest/servicesdirectory/edit"
        params["f"] = "json"
        return self._post(url=url,
                          param_dict=params,
                          securityHandler=self._securityHandler,
                          proxy_port=self._proxy_port,
                          proxy_url=self._proxy_url)
    @property
    #----------------------------------------------------------------------
    def deployment(self):
        """
        Deployment configuration resource that can control
        the load-balancing functionality between GIS server machines.
        """
        url = self._url + "/deployment"
        params = {
            "f" : "json"
        }
        return self._get(url=url,
                        param_dict=params,
                        securityHandler=self._securityHandler,
                        proxy_port=self._proxy_port,
                        proxy_url=self._proxy_url)
    #----------------------------------------------------------------------
    def updateDeploymentConfiguration(self,singleClusterMode):
        """
        Updates deployment configuration setting singleClusterMode property
        either to true or false; this defines whether you will be able to
        create new clusters and use load balancing. This property is added
        in ArcGIS Server 10.4.

        Inputs:
            singleClusterMode - true/false. To enable load balancing, set
                                this property to false. Updating this property
                                will restart all machines in the site.
        """
        url = self._url + "/deployment/update"
        params = {
            "f" : "json",
            "deploymentConfig": singleClusterMode
        }
        return self._post(url=url,
                          param_dict=params,
                          securityHandler=self._securityHandler,
                          proxy_port=self._proxy_port,
                          proxy_url=self._proxy_url)
########################################################################
class ConfigurationStore(BaseAGSServer):
    """"""
    _url = None
    _proxy_url = None
    _json = None
    _proxy_port = None
    _securityHandler = None
    _json_dict = None
    _type = None
    _connectionString = None
    _class = None
    _localRepositoryPath = None
    _status = None
    #----------------------------------------------------------------------
    def __init__(self, url,
                 securityHandler,
                 proxy_url=None,
                 proxy_port=None,
                 initialize=False):
        """Constructor"""
        self._url = url
        self._securityHandler = securityHandler
        self._proxy_url = proxy_url
        self._proxy_port = proxy_port
        if initialize:
            self.__init()
    #----------------------------------------------------------------------
    def __init(self):
        """ populates server admin information """
        params = {
            "f" : "json"
        }
        json_dict = self._get(url=self._url, param_dict=params,
                                 securityHandler=self._securityHandler,
                                 proxy_url=self._proxy_url,
                                 proxy_port=self._proxy_port)
        self._json_dict = json_dict
        self._json = json.dumps(json_dict)
        attributes = [attr for attr in dir(self)
                      if not attr.startswith('__') and \
                      not attr.startswith('_')]
        for k,v in json_dict.items():
            if k == "class":
                self._class = v
            elif k in attributes:
                setattr(self, "_"+ k, json_dict[k])
            else:
                print( k, " - attribute not implemented in Configuration store.")
            del k
            del v
    #----------------------------------------------------------------------
    def __str__(self):
        """returns the object as a string"""
        return json.dumps(self._json)
    #----------------------------------------------------------------------
    @property
    def type(self):
        """gets the configuration store type"""
        if self._type is None:
            self.__init()
        return self._type
    #----------------------------------------------------------------------
    @property
    def connectionString(self):
        """gets the connection string"""
        if self._connectionString is None:
            self.__init()
        return self._connectionString
    #----------------------------------------------------------------------
    @property
    def classValue(self):
        """gets the class value"""
        if self._class is None:
            self.__init()
        return self._class
    #----------------------------------------------------------------------
    @property
    def localRepositoryPath(self):
        """gets the class value"""
        if self._localRepositoryPath is None:
            self.__init()
        return self._localRepositoryPath
    #----------------------------------------------------------------------
    @property
    def status(self):
        """gets the status value"""
        if self._status is None:
            self.__init()
        return self._status
    #----------------------------------------------------------------------
    def edit(self, typeValue,
             connectionString,
             move=True,
             runAsync=False):
        """
        You can use this operation to update the configuration store.
        Typically, this operation is used to change the location of the
        store.
        When ArcGIS Server is installed, the default configuration store
        uses local paths. As the site grows (more server machines are
        added), the location of the store must be updated to use a shared
        file system path. On the other hand, if you know at the onset that
        your site will have two or more server machines, you can start from
        a shared path while creating a site and skip this step altogether.

        Inputs:
           typeValue - Type of the configuration store. Values: FILESYSTEM
           connectionString - A file path or connection URL to the physical
            location of the store.
           move - default True - A boolean to indicate if you want to move
            the content of the current store to the new store.
           runAsync - default False - Decides if this operation must run
            asynchronously.
        """
        url = self._url + "/edit"
        params = {
            "f" : "json",
            "type" : typeValue,
            "connectionString" : connectionString,
            "move" : move,
            "runAsync" : runAsync
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    def recover(self,runAsync=False):
        """
        If the shared configuration store for a site is unavailable, a site
        in read-only mode will operate in a degraded capacity that allows
        access to the ArcGIS Server Administrator Directory. You can recover
        a site if the shared configuration store is permanently lost. The
        site must be in read-only mode, and the site configuration files
        must have been copied to the local repository when switching site
        modes. The recover operation will copy the configuration store from
        the local repository into the shared configuration store location.
        The copied local repository will be from the machine in the site
        where the recover operation is performed.
        Inputs:
           runAsync - default False - Decides if this operation must run
            asynchronously.
        """
        url = self._url + "/recover"
        params = {
            "f" : "json",
            "runAsync" : runAsync
        }
        return self._post(url=url,
                          param_dict=params,
                          securityHandler=self._securityHandler,
                          proxy_url=self._proxy_url,
                          proxy_port=self._proxy_port)
########################################################################
class Jobs(BaseAGSServer):
    """
    This resource is a collection of all the administrative jobs
    (asynchronous operations) created within your site. When operations
    that support asynchronous execution are run, the server creates a new
    job entry that can be queried for its current status and messages.
    """
    _json = None
    _jobs = None
    _json_dict = None
    _url = None
    _securityHandler = None
    _proxy_url = None
    _proxy_port = None

    #----------------------------------------------------------------------
    def __init__(self, url,
                 securityHandler,
                 proxy_url=None,
                 proxy_port=None,
                 initialize=False):
        """Constructor"""
        self._url = url
        self._securityHandler = securityHandler
        self._proxy_url = proxy_url
        self._proxy_port = proxy_port
        if initialize:
            self.__init()
    #----------------------------------------------------------------------
    def __init(self):
        """ populates server admin information """
        params = {
            "f" : "json"
        }
        json_dict = self._get(url=self._url, param_dict=params,
                                 securityHandler=self._securityHandler,
                                 proxy_url=self._proxy_url,
                                 proxy_port=self._proxy_port)
        self._json = json.dumps(json_dict)
        self._json_dict = json_dict
        attributes = [attr for attr in dir(self)
                      if not attr.startswith('__') and \
                      not attr.startswith('_')]
        for k,v in json_dict.items():
            if k in attributes:
                setattr(self, "_"+ k, json_dict[k])
            else:
                print( k, " - attribute not implemented in Jobs.")
            del k
            del v
    #----------------------------------------------------------------------
    def __str__(self):
        """returns the object as a string"""
        return json.dumps(self._json)
    #----------------------------------------------------------------------
    @property
    def jobs(self):
        """gets the job ids"""
        if self._jobs is None:
            self.__init()
        return self._jobs
    #----------------------------------------------------------------------
    def getJobDetails(self, jobId):
        """
        A job represents the asynchronous execution of an operation. You
        can acquire progress information by periodically querying the job.

        Inputs:
           jobId - id of the job
        """
        url = self._url + "/%s" % jobId
        params = {
            "f" : "json"
        }
        return self._get(url=url,
                            param_dict=params,
                            securityHandler=self._securityHandler,
                            proxy_url=self._proxy_url,
                            proxy_port=self._proxy_port)
########################################################################
class ServerProperties(BaseAGSServer):
    """
    The Server has configuration parameters that can be govern some of its
    intricate behavior. The Server Properties resource is a container for
    such properties. These properties are available to all server objects
    and extensions through the server environment interface.
    The properties include:
     CacheSizeForSecureTileRequests - An integer that specifies the
      number of users whose token information will be cached. This
      increases the speed of tile retrieval for cached services. If not
      specified, the default cache size is 200,000. Both REST and SOAP
      services honor this property. You'll need to manually restart
      ArcGIS Server in order for this change to take effect.
     DisableAdminDirectoryCache - Disables browser caching of the
      Administrator Directory pages. The default is false. To disable
      browser caching, set this property to true.
     disableIPLogging - When a possible cross-site request forgery
      (CSRF) attack is detected, the server logs a message containing
      the possible IP address of the attacker. If you do not want IP
      addresses listed in the logs, set this property to true. Also,
      HTTP request referrers are logged at FINE level by the REST and
      SOAP handlers unless this property is set to true.
     javaExtsBeginPort - Specifies a start port of the port range used
      for debugging Java server object extensions.
      Example: 8000
     javaExtsEndPort - Specifies an end port of the port range used for
      debugging Java server object extensions.
      Example: 8010
     localTempFolder - Defines the local folder on a machine that can
      be used by GIS services and objects. If this property is not
      explicitly set, the services and objects will revert to using the
      system's default temporary directory.

      Note:
      If this property is used, you must create the temporary directory
      on every server machine in the site. Example: /tmp/arcgis.

      messageFormat - Defines the transmission protocol supported by
       the services catalog in the server.
       Values: esriServiceCatalogMessageFormatBin,
               esriServiceCatalogMessageFormatSoap,
               esriServiceCatalogMessageFormatSoapOrBin
      messageVersion - Defines the version supported by the services
       catalog in the server. Example: esriArcGISVersion101
      PushIdentityToDatabase - Propogates the credentials of the logged
       -in user to make connections to an Oracle database. This
       property is only supported for use with Oracle databases.
       Values: true | false
      suspendDuration - Specifies the duration for which the ArcGIS
       service hosting processes should suspend at startup. This
       duration is specified in milliseconds. This is an optional
       property that takes effect when suspendServiceAtStartup is set
       to true. If unspecified and suspension of service at startup is
       requested, then the default suspend duration is 30 seconds.
       Example: 10000 (meaning 10 seconds)
      suspendServiceAtStartup - Suspends the ArcGIS service hosting
       processes at startup. This will enable attaching to those
       processes and debugging code that runs early in the lifecycle of
       server extensions soon after they are instantiated.
       Values: true | false
      uploadFileExtensionWhitelist - This specifies what files are
       allowed to be uploaded through the file upload API by
       identifying the allowable extensions. It is a list of comma
       separated extensions without dots. If this property is not
       specified a default list is used. This is the default list: soe,
       sd, sde, odc, csv, txt, zshp, kmz, and geodatabase.

     Note:
     Updating this list overrides the default list completely. This
     means if you set this property to a subset of the default list
     then only those items in the subset will be accepted for upload.
     Example: sd, so, sde, odc.

     uploadItemInfoFileExtensionWhitelist - This specifies what files
      are allowed to be uploaded through the service iteminfo upload
      API by identifying the allowable extensions. It should be a list
      of comma separated extensions without dots. If this property is
      not specified a default list is used. This is the default list:
      xml, img, png, gif, jpg, jpeg, bmp.

    Note:
    This list overrides the default list completely. This means if you
    set this property to a subset of the default list then only those
    items in the subset will be accepted for upload. Example: png, svg,
    gif, jpg, tiff, bmp.

    WebContextURL - Defines the web front end as seen by your users.
     Example: http://mycompany.com/gis
    """
    _url = None
    _json = None
    _proxy_url = None
    _proxy_port = None
    _securityHandler = None
    #----------------------------------------------------------------------
    def __init__(self, url,
                 securityHandler,
                 proxy_url=None,
                 proxy_port=None,
                 initialize=False):
        """Constructor"""
        if url.lower().endswith('/properties'):
            self._url = url
        else:
            self._url = url + "/properties"
        self._securityHandler = securityHandler
        self._proxy_port = proxy_port
        self._proxy_url = proxy_url
        if initialize:
            self.__init()
    #----------------------------------------------------------------------
    def __init(self):
        """ populates server admin information """
        params = {
            "f" : "json"
        }
        json_dict = self._get(url=self._url, param_dict=params,
                                 securityHandler=self._securityHandler,
                                 proxy_url=self._proxy_url,
                                 proxy_port=self._proxy_port)
        self._json = json.dumps(json_dict)
        attributes = [attr for attr in dir(self)
                      if not attr.startswith('__') and \
                      not attr.startswith('_')]
        for k,v in json_dict.items():
            if k in attributes:
                setattr(self, "_"+ k, json_dict[k])
            else:
                print( k, " - attribute not implemented in ServerProperties.")
            del k
            del v
    #----------------------------------------------------------------------
    def __str__(self):
        """returns the object as a string"""
        return json.dumps(self._json)
    #----------------------------------------------------------------------
    def updateServerProperties(self, properties):
        """
        This operation allows you to update the server property
        """
        url = self._url + "/update"
        params = {
            "f" : "json",
            "properties" : properties
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port)
########################################################################
class ServerDirectory(BaseAGSServer):
    """
    Server directories are used by GIS services as a location to output
    items such as map images, tile caches, and geoprocessing results. In
    addition, some directories contain configurations that power the GIS
    services.
    In a Site with more than one server machine these directories must be
    available on network shares, accessible to every machine in the site.

    The following directory types can be registered with the server:
     Output - Stores various information generated by services, such as map
      images. Instances: One or more
     Cache - Stores tile caches used by map, globe, and image services for
      rapid performance. Instances: One or more
     Jobs - Stores results and other information from geoprocessing
      services. Instances: One or more
     System - Stores files that are used internally by the GIS server.
      Instances: One Server directories that contain output of various GIS
      services can be periodically cleaned to remove old unused files. By
      using the cleanup mode and maximum file age parameters, you control
      when when you would like the files in these directories to be
      cleaned.

    All the output server directories are automatically virtualized (they
    can be accessed over a URL) for you through the ArcGIS Server REST API.
    """
    _json = None
    _securityHandler = None
    _proxy_port = None
    _proxy_url = None
    _url = None
    _name = None
    _physicalPath = None
    _directoryType = None
    _cleanupMode = None
    _maxFileAge = None
    _description = None
    _useLocalDir = None
    _localDirectoryPath = None
    _virtualPath = None
    #----------------------------------------------------------------------
    def __init__(self, url,
                 securityHandler,
                 proxy_url=None,
                 proxy_port=None,
                 initialize=False):
        """Constructor"""
        self._url = url
        self._securityHandler = securityHandler
        self._proxy_port = proxy_port
        self._proxy_url = proxy_url
        if initialize:
            self.__init()
    #----------------------------------------------------------------------
    def __init(self):
        """ populates server admin information """
        params = {
            "f" : "json"
        }
        json_dict = self._get(url=self._url, param_dict=params,
                                 securityHandler=self._securityHandler,
                                 proxy_url=self._proxy_url,
                                 proxy_port=self._proxy_port)
        self._json = json.dumps(json_dict)
        attributes = [attr for attr in dir(self)
                      if not attr.startswith('__') and \
                      not attr.startswith('_')]
        for k,v in json_dict.items():
            if k in attributes:
                setattr(self, "_"+ k, json_dict[k])
            else:
                print( k, " - attribute not implemented in ServerDirectory.")
            del k
            del v
    #----------------------------------------------------------------------
    def __str__(self):
        """returns the object as a string"""
        return json.dumps(self._json)
    #----------------------------------------------------------------------
    @property
    def name(self):
        """gets the directory name"""
        if self._name is None:
            self.__init()
        return self._name
    #----------------------------------------------------------------------
    @property
    def physicalPath(self):
        """gets the physical path"""
        if self._physicalPath is None:
            self.__init()
        return self._physicalPath
    #----------------------------------------------------------------------
    @property
    def directoryType(self):
        """gets the directoryType value"""
        if self._directoryType is None:
            self.__init()
        return self._directoryType
    #----------------------------------------------------------------------
    @property
    def cleanupMode(self):
        """gets the cleanupMode value"""
        if self._cleanupMode is None:
            self.__init()
        return self._cleanupMode
    #----------------------------------------------------------------------
    @property
    def maxFileAge(self):
        """gets the maxFileAge value"""
        if self._maxFileAge is None:
            self.__init()
        return self._maxFileAge
    #----------------------------------------------------------------------
    @property
    def description(self):
        """gets the description value"""
        if self._description is None:
            self.__init()
        return self._description
    #----------------------------------------------------------------------
    @property
    def useLocalDir(self):
        """gets the description value"""
        if self._useLocalDir is None:
            self.__init()
        return self._useLocalDir
    #----------------------------------------------------------------------
    @property
    def localDirectoryPath(self):
        """gets the description value"""
        if self._localDirectoryPath is None:
            self.__init()
        return self._localDirectoryPath
    #----------------------------------------------------------------------
    @property
    def virtualPath(self):
        """gets the virtualPath value"""
        if self._virtualPath is None:
            self.__init()
        return self._virtualPath
    #----------------------------------------------------------------------
    def edit(self,
             physicalPath,
             cleanupMode,
             maxFileAge,
             description):
        """
        The server directory's edit operation allows you to change the path
        and clean up properties of the directory. This operation updates
        the GIS service configurations (and points them to the new path)
        that are using this directory, causing them to restart. It is
        therefore recommended that any edit to the server directories be
        performed when the server is not under load.
        This operation is mostly used when growing a single machine site to
        a multiple machine site configuration, which requires that the
        server directories and configuration store be put on a
        network-accessible file share.

        Inputs:
           physicalPath - The absolute physical path of the server
            directory.
           cleanupMode - Defines if files in the server directory needs to
            be cleaned up. The default is NONE.
           maxFileAge - Defines how long a file in the directory needs to
            be kept before it is deleted.
           description - An optional description for the server directory
        """
        url = self._url + "/edit"
        params = {
            "f" : "json",
            "physicalPath" : physicalPath,
            "cleanupMode" : cleanupMode,
            "maxFileAge" : maxFileAge,
            "description" : description
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    def clean(self):
        """
        Cleans the content (files and folders) within the directory that
        have passed their expiration date. Every server directory has the
        max file age and cleanup mode parameter that govern when a file
        created inside is supposed to be cleaned up. The server directory
        cleaner automatically cleans up the content within server
        directories at regular intervals. However, you can explicitly clean
        the directory by invoking this operation.
        """
        url = self._url + "/clean"
        params = {
            "f" : "json"
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port)
    #----------------------------------------------------------------------
    def unregisterDirectory(self):
        """
        Unregisters a server directory. Once a directory has been
        unregistered, it can no longer be referenced (used) from within a
        GIS service.
        """
        url = self._url + "/unregister"
        params = {
            "f" : "json"
        }
        return self._post(url=url,
                             param_dict=params,
                             securityHandler=self._securityHandler,
                             proxy_url=self._proxy_url,
                             proxy_port=self._proxy_port)


# -*- coding=utf-8 -*-

from __future__ import division, print_function

import socket
import select
import sys
sys.path.append('..')
from data.device_data import Packet
import time

class Subscriber(object):
    def __init__(self, host, port, buffer_size=1024):
        self.buffer_size = buffer_size
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        connected = False
        print('Waiting for connection...')
        while not connected:
            try:
                self.sock.connect((host, port))
                connected = True
            except:
                time.sleep(0.1)

        print('Connected to publisher')
        self.callbacks = []

    def run(self):
        self.running = True
        while self.running:
            socket_list = [self.sock]
            to_read, to_write, in_error = select.select(socket_list, [], [])
            for sock in to_read:
                if sock == self.sock:
                    data = sock.recv(self.buffer_size)
                    if data:
                        print('Data received')
                        self.execute_callbacks(data)
                        #print(Packet.unpack(data))

    def execute_callbacks(self, data):
        for c in self.callbacks:
            c(data)

    def add_callback(self, callback):
        self.callbacks.append(callback)

if __name__ == '__main__':
    host = 'localhost'
    port = 5000
    #Subscriber(host, port).run()
    s = Subscriber(host, port)

    def print_data(data):
        print(Packet.unpack(data))

    s.add_callback(print_data)
    s.run()


import glob
import os
from osgeo import gdal
import sys

# python3 utils/create-cog.py 32000000,16000000,8000000

# set scale
scales = sys.argv[1].split(",")
int_scales = [int(x) for x in scales]
sorted_scales = sorted(int_scales, reverse=False)
root_scale = min(int_scales)

# Process scales to tifs
ovr_ext = ".ovr"
ovr_name = f"cog-outputs/{int(root_scale)}k.tif"

for scale in sorted_scales:
    vrt_list = []
    if scale != root_scale:
        for f in glob.glob(f'cog-outputs/**/{str(scale)}_gtiff_images.tif', recursive=True):
            vrt_list.append(f)

        # Build VRT
        gdal.BuildVRT(
            f"cog-outputs/{str(scale)}k.vrt",
            vrt_list
        )

        creation_options = [
            "PROFILE=BASELINE",
            "BIGTIFF=YES"
        ]

        # Export VRT as Image
        print(f"Process overview: {scale}")
        ovr_name = f"{ovr_name}{ovr_ext}"
        gdal.Translate(
            ovr_name,
            f"cog-outputs/{str(scale)}k.vrt",
            format = "GTiff",
            resampleAlg = "bilinear",
            creationOptions = creation_options,
            callback = gdal.TermProgress_nocb
        )

    # Process Root Scale
    elif scale == root_scale:
        for f in glob.glob(f'cog-outputs/**/{str(scale)}_gtiff_images.tif', recursive=True):
            vrt_list.append(f)

        # Build VRT
        gdal.BuildVRT(
            f"cog-outputs/{str(scale)}k.vrt",
            vrt_list
        )

        creation_options = [
            "TILED=YES",
            "COMPRESS=JPEG",
            "BIGTIFF=YES"
        ]

        # Export VRT as Image
        gdal.Translate(
            f"cog-outputs/{str(scale)}k.tif",
            f"cog-outputs/{str(scale)}k.vrt",
            format = "GTiff",
            resampleAlg = "bilinear",
            creationOptions = creation_options,
            callback = gdal.TermProgress_nocb
        )

    else:
        print("Scale is not an int")

# create cog
creation_options = [
    "COMPRESS=JPEG",
    "BIGTIFF=YES",
    "NUM_THREADS=ALL_CPUS"
]

# Save stack as COG
print("Creating COG")
gdal.SetConfigOption('RESAMPLING', 'BILINEAR')
gdal.Translate(
    f"cog-outputs/{str(root_scale)}k_cog.tif",
    f"cog-outputs/{str(root_scale)}k.tif",
    format = "COG",
    creationOptions = creation_options,
    callback = gdal.TermProgress_nocb
)


import os
import pytest
from flask import _request_ctx_stack
from app.factory import create_app
from app.models import db as _db

@pytest.fixture(scope='session')
def app(request):
    """Session-wide test `Flask` application."""
    app = create_app(os.getcwd() + '/app_config.py')

    # Establish an application context before running the tests.
    ctx = app.app_context()
    ctx.push()

    def teardown():
        ctx.pop()

    request.addfinalizer(teardown)
    return app

@pytest.fixture(scope='function')
def session(app, request):
    """Creates a fresh database and
    new session for a test.
    """
    _db.app = app
    _db.create_all()

    connection = _db.engine.connect()

    options = dict(bind=connection)
    session = _db.create_scoped_session(options=options)

    _db.session = session

    def teardown():
        connection.close()
        session.remove()
        _db.drop_all()

    request.addfinalizer(teardown)
    return session

@pytest.fixture(scope='session')
def client(request, app):
    return app.test_client()

@pytest.fixture(scope='session')
def request_ctx(request, app):
    ctx = app.test_request_context()
    ctx.push()

    def teardown():
        if _request_ctx_stack.top and _request_ctx_stack.top.preserved:
            _request_ctx_stack.top.pop()
        ctx.pop()

    request.addfinalizer(teardown)
    return ctx


import logging
from datetime import datetime

from ckan.lib.munge import munge_title_to_name
from ckan.model.meta import *
from ckan.model.types import make_uuid
from ckan.model.core import *
from ckan.model.resource import Resource
from ckan.model.domain_object import DomainObject

from sqlalchemy.orm import backref, relation

__all__ = [
    'ResourceRelatedElement', 'resource_related_element_table',
]

resource_related_element_table = None

def setup():
    if resource_related_element_table is None:
        create_apps_tables()
    metadata.create_all()

resource_related_element_table = Table('resource_related_element',metadata,
        Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
        Column('resource_id', types.UnicodeText,
            ForeignKey('resource.id')),
        Column('name', types.Unicode(), nullable=True),
        Column('title', types.Unicode(), nullable=True),
        Column('url', types.UnicodeText, nullable=True),
        Column('embed_code', types.UnicodeText),
        Column('created', DateTime, default=datetime.datetime.now),
        Column('updated', DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now),
    )

class ResourceRelatedElement(DomainObject):
    @classmethod
    def generate_name(cls, title):
        return _generate_name(cls,title)
## Mappers

mapper(ResourceRelatedElement, resource_related_element_table, properties={
    'resource':orm.relation(Resource,
        # all resources including deleted
        # formally resource_related_stuff
        backref=orm.backref('related_elements',
                            cascade='all, delete',
                            ),
                       )
    },
)

def create_apps_tables():
    global resource_related_element_table


#! /usr/bin/env python
# -*- encoding:utf-8 -*-
import os
import sys
import re
import logging
import pysam
import time
import subprocess

from multiprocessing import Pool
from collections import defaultdict
from itertools import izip_longest

import utils

LOGGER = logging.getLogger('CIRIquant')
PREFIX = re.compile(r'(.+)[/_-][12]')

class BedParser(object):
    """
    Class for parsing circRNA information in bed file
    """

    def __init__(self, content):
        self.chrom = content[0]
        self.start = int(content[1])
        self.end = int(content[2])
        self.circ_id = content[3]
        self.strand = content[5]
        self.length = self.end - self.start + 1

def load_bed(fname):
    """
    Load Back-Spliced Junction Sites in bed file

    Parameters
    -----
    fname : str
        input file name

    Returns
    -----
    dict
        divide all circRNAs into different chromsomes

    """
    circ_info = defaultdict(dict)
    with open(fname, 'r') as f:
        for line in f:
            content = line.rstrip().split('\t')
            parser = BedParser(content)
            circ_info[parser.chrom][parser.circ_id] = parser
    return circ_info

def update_info(circ_info, rnaser_file):
    """
    Add information of RNase R circRNAs
    """
    circ_exp = {}
    LOGGER.info('RNase R file: {}'.format(rnaser_file))
    with open(rnaser_file, 'r') as f:
        header = {}
        for line in f:
            if line.startswith('##'):
                key, value = line.rstrip().strip('#').split(':')
                header.update({key: value})
                continue

            content = line.rstrip().split('\t')
            tmp_parser = GTFParser(content)
            circ_exp[tmp_parser.attr['circ_id']] = {
                'bsj': float(tmp_parser.attr['bsj']),
                'fsj': float(tmp_parser.attr['fsj']),
                'ratio': 2 * float(tmp_parser.attr['bsj']) / (2 * float(tmp_parser.attr['bsj']) + float(tmp_parser.attr['fsj']))
            }
            parser = BedParser([
                tmp_parser.chrom,
                tmp_parser.start,
                tmp_parser.end,
                tmp_parser.attr['circ_id'],
                '.',
                tmp_parser.strand,
            ])
            if parser.chrom in circ_info and parser.circ_id in circ_info[parser.chrom]:
                continue
            circ_info[parser.chrom].update({parser.circ_id: parser})

    return circ_exp, (int(header['Total_Reads']), int(header['Mapped_Reads']), int(header['Circular_Reads']))

def load_fai(fname):
    """
    Load fai index of fasta created by samtools

    Parameters
    -----
    fname : str
        input fai file name

    Returns
    -----
    dict
         chromsome and start / end position in file

    """
    faidx = {}
    with open(fname, 'r') as f:
        for line in f:
            content = line.rstrip().split('\t')
            chrom, length, start, eff_length, line_length = content
            shift_length = int(length) * int(line_length) / int(eff_length)
            faidx[chrom] = [int(start), shift_length]
    return faidx

def extract_seq(fasta, start, length):
    """
    Extract sequence from fasta according to given position

    Parameters
    -----
    fasta : str
        file name of fasta
    start : int
        offset of chrosome
    length : int
        length of chromsome sequence

    Returns
    -----
    str
        sequence from start to start + length

    """
    with open(fasta, 'r') as f:
        f.seek(start, 0)
        seq = f.read(length)
        seq = re.sub('\n', '', seq)
    return seq

def generate_index(log_file, circ_info, circ_fasta):
    """
    Generate pseudo circular index

    Parameters
    -----
    log_file : str
        file name of log file, used for subprocess.PIPE
    circ_info : dict
        back-spliced junction sites
    circ_fasta :
        output fasta file name

    Returns
    -----
    dict
        chromosomes used in BSJ sites

    """

    from logger import ProgressBar

    fai = utils.FASTA + '.fai'
    if not os.path.exists(fai):
        LOGGER.debug('Indexing FASTA')
        index_cmd = '{} faidx {}'.format(utils.SAMTOOLS, utils.FASTA)
        with open(log_file, 'a') as log:
            LOGGER.debug(index_cmd)
            subprocess.call(index_cmd, shell=True, stderr=log, stdout=log)
    fasta_index = load_fai(fai)

    LOGGER.info('Extract circular sequence')
    prog = ProgressBar()
    prog.update(0)
    cnt = 0
    with open(circ_fasta, 'w') as out:
        for chrom in sorted(circ_info.keys()):
            prog.update(100 * cnt / len(circ_info))
            cnt += 1
            if chrom not in fasta_index:
                sys.exit('Unconsistent chromosome id: {}'.format(chrom))

            chrom_start, chrom_length = fasta_index[chrom]
            chrom_seq = extract_seq(utils.FASTA, chrom_start, chrom_length)

            chrom_circ = circ_info[chrom]
            for circ_id in chrom_circ:
                parser = chrom_circ[circ_id]
                circ_seq = chrom_seq[parser.start - 1:parser.end] * 2
                if circ_seq.count('N') > len(circ_seq) * 0.5 or len(circ_seq) == 0:
                    continue
                out.write('>{}'.format(parser.circ_id) + '\n')
                out.write(circ_seq + '\n')
    prog.update(100)

    return fasta_index

def build_index(log_file, thread, pseudo_fasta, outdir, prefix):
    """
    Build hisat2 index for pseudo circular index

    Returns
    -----
    str
        index file name used in denovo mapping

    """
    LOGGER.info('Building circular index ..')
    denovo_index = '{}/circ/{}_index'.format(outdir, prefix)

    build_cmd = '{}-build -p {} -f {} {}'.format(
        utils.HISAT2,
        thread,
        pseudo_fasta,
        denovo_index
    )

    with open(log_file, 'a') as log:
        LOGGER.debug(build_cmd)
        subprocess.call(build_cmd, shell=True, stderr=log, stdout=log)

    return denovo_index

def denovo_alignment(log_file, thread, reads, outdir, prefix):
    """
    Call hisat2 to read re-alignment

    Returns
    -----
    str
        Output bam file

    """
    LOGGER.info('De novo alignment for circular RNAs ..')
    denovo_bam = '{}/circ/{}_denovo.bam'.format(outdir, prefix)
    sorted_bam = '{}/circ/{}_denovo.sorted.bam'.format(outdir, prefix)

    align_cmd = '{} -p {} --dta -q -x {}/circ/{}_index -1 {} -2 {} | {} view -bS > {}'.format(
        utils.HISAT2,
        thread,
        outdir,
        prefix,
        reads[0],
        reads[1],
        utils.SAMTOOLS,
        denovo_bam,
    )

    sort_cmd = '{} sort --threads {} -o {} {}'.format(
        utils.SAMTOOLS,
        thread,
        sorted_bam,
        denovo_bam,
    )

    index_cmd = '{} index -@ {} {}'.format(
        utils.SAMTOOLS,
        thread,
        sorted_bam,
    )

    with open(log_file, 'a') as log:
        LOGGER.debug(align_cmd)
        subprocess.call(align_cmd, shell=True, stderr=log, stdout=log)

        LOGGER.debug(sort_cmd)
        subprocess.call(sort_cmd, shell=True, stderr=log, stdout=log)

        LOGGER.debug(index_cmd)
        subprocess.call(index_cmd, shell=True, stderr=log, stdout=log)
    # wait for samtools index to correct time stamp
    time.sleep(5)
    return sorted_bam

def grouper(iterable, n, fillvalue=None):
    """
    Collect data info fixed-length chunks or blocks
    grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    """

    args = [iter(iterable)] * n
    return izip_longest(*args, fillvalue=None)

def proc_denovo_bam(bam_file, thread, circ_info, threshold, lib_type):
    """
    Extract BSJ reads in denovo mapped bam file

    Returns
    -----
    dict
        query_name -> mate_id -> pysam.AlignSegment

    """

    LOGGER.info('Detecting reads containing Back-splicing signals')

    sam = pysam.AlignmentFile(bam_file, 'rb')
    header = sam.header['SQ']
    sam.close()

    pool = Pool(thread, denovo_initializer, (bam_file, circ_info, threshold, ))
    jobs = []
    chunk_size = max(500, len(header) / thread + 1)
    for circ_chunk in grouper(header, chunk_size):
        jobs.append(pool.apply_async(denovo_worker, (circ_chunk, lib_type, )))
    pool.close()
    pool.join()

    cand_reads = defaultdict(dict)
    for job in jobs:
        tmp_cand = job.get()
        for read_id, mate_id, circ_id, blocks, cigartuples in tmp_cand:
            cand_reads[read_id][mate_id] = (circ_id, blocks, cigartuples)

    return cand_reads

BAM = None
THRESHOLD = None
CIRC = None

def denovo_initializer(infile, circ_info, threshold):
    """
    Initializer for passing bam file name
    """
    global BAM, CIRC, THRESHOLD
    BAM, CIRC, THRESHOLD = infile, circ_info, threshold

def denovo_worker(circ_chunk, lib_type):
    """
    Find candidate reads with junction signal

    Parameters
    -----
    circ_chunk : list
        list of Pysam header to process

    Returns
    -----
    list
        pysam.AlignedSegment, candidate reads with junction signal

    """
    #TODO: add support for stranded specific RNA_seq libraries
    sam = pysam.AlignmentFile(BAM, 'rb')
    cand_reads = []
    for d in circ_chunk:
        if d is None:
            continue
        circ_id, junc_site = d['SN'], int(d['LN']) / 2
        contig = circ_id.split(':')[0]
        parser = CIRC[contig][circ_id]

        tmp_cand = []
        circ_reads = defaultdict(dict)
        for read in sam.fetch(circ_id, multiple_iterators=True):
            if read.is_unmapped or read.is_secondary or read.is_supplementary:
                continue

            read_strand = '-' if read.is_reverse else '+'
            if lib_type == 0:
                pass
            elif lib_type == 1:
                if read.is_read1 - read.is_read2 > 0:
                    if read_strand != parser.strand:
                        continue
                else:
                    if read_strand == parser.strand:
                        continue
            elif lib_type == 2:
                if read.is_read1 - read.is_read2 > 0:
                    if read_strand == parser.strand:
                        continue
                else:
                    if read_strand != parser.strand:
                        continue

            circ_reads[query_prefix(read.query_name)][read.is_read1 - read.is_read2] = 1
            if read.mapping_quality <= 10:
                continue
            if read.get_overlap(junc_site - THRESHOLD, junc_site + THRESHOLD) >= THRESHOLD * 2:
                tmp_cand.append((read.query_name, read.is_read1 - read.is_read2, circ_id, read.get_blocks(), read.cigartuples))

        for qname, mate_id, circ_id, blocks, cigartuples in tmp_cand:
            if -1 * mate_id in circ_reads[query_prefix(qname)]:
                cand_reads.append((qname, mate_id, circ_id, blocks, cigartuples))
    sam.close()

    return cand_reads

def proc_genome_bam(bam_file, thread, circ_info, cand_reads, threshold, tmp_dir, is_no_fsj):
    """
    Extract FSJ reads and check BSJ reads alignment information

    Returns
    -----
    dict
        bsj reads of circRNAs, pair_id -> mate_id -> circ_id
    dict
        fsj reads of circRNAs, pair_id -> mate_id -> circ_id

    """
    import cPickle
    LOGGER.info('Detecting FSJ reads from genome alignment file')

    sam = pysam.AlignmentFile(bam_file, 'rb')
    header = sam.header['SQ']
    sam.close()

    pool = Pool(thread, genome_initializer, (bam_file, circ_info, cand_reads, threshold))
    jobs = []
    for chrom_info in header:
        jobs.append(pool.apply_async(genome_worker, (chrom_info['SN'], tmp_dir, is_no_fsj, )))
    pool.close()
    pool.join()

    fp_bsj = defaultdict(dict)
    fsj_reads = defaultdict(dict)
    cand_to_genome = []

    for job in jobs:
        tmp = job.get()
        if tmp is None:
            continue
        res = tmp if isinstance(tmp, dict) else cPickle.load(open(tmp, 'rb'))
        chrom_fp_bsj, chrom_fsj, chrom_cand = res['fp_bsj'], res['fsj_reads'], res['cand_to_genome']
        for pair_id, mate_id in chrom_fp_bsj:
            fp_bsj[pair_id][mate_id] = 1
        for pair_id, mate_id, circ_id in chrom_fsj:
            fsj_reads[pair_id][mate_id] = circ_id
        cand_to_genome += chrom_cand

    circ_bsj = defaultdict(dict)
    circ_fsj = defaultdict(dict)
    for pair_id in cand_reads:
        for mate_id, (circ_id, blocks, cigartuples) in cand_reads[pair_id].iteritems():
            if pair_id in fp_bsj and mate_id in fp_bsj[pair_id]:
                continue
            circ_bsj[circ_id].update({query_prefix(pair_id): 1})

    for pair_id in fsj_reads:
        for mate_id, circ_id in fsj_reads[pair_id].iteritems():
            if pair_id in cand_reads and mate_id in cand_reads[pair_id] and not (pair_id in fp_bsj and mate_id in fp_bsj[pair_id]):
                continue
            circ_fsj[circ_id].update({query_prefix(pair_id): 1})

    # sam = pysam.AlignmentFile(bam_file, 'rb')
    # cand_sam = pysam.AlignmentFile(bam_file + '.fsj', 'w', template=sam)
    # cand_sam.close()
    # sam.close()
    #
    # with open(bam_file + '.fsj', 'w') as out:
    #     for read in cand_to_genome:
    #         out.write(read + '\n')
    return circ_bsj, circ_fsj

BSJ = None

def genome_initializer(bam_file, circ_info, cand_bsj, threshold):
    """
    Initializer for passing bam file name and circRNA_info
    """
    global BAM, CIRC, THRESHOLD, BSJ
    BAM, CIRC, BSJ, THRESHOLD = bam_file, circ_info, cand_bsj, threshold

def genome_worker(chrom, tmp_dir, is_no_fsj):
    """
    Find FSJ reads and re-check BSJ reads

    Parameters
    -----
    chrom : str
        chromosme or scaffold name for process

    Returns
    -----
    list
        false positive reads information,  (query_name, mate_id)
    list
        fsj_reads of circRNAs, (query_name, mate_id, circ_id)

    """
    import cPickle

    if chrom not in CIRC:
        return None

    sam = pysam.AlignmentFile(BAM, 'rb')
    cand_to_genome = []

    fp_bsj = []
    for read in sam.fetch(chrom, multiple_iterators=True):
        # If Reads is bsj candidate
        if read.is_unmapped:
            continue
        if read.query_name not in BSJ or read.is_read1 - read.is_read2 not in BSJ[read.query_name]:
            continue
        circ_id, blocks, cigartuples = BSJ[read.query_name][read.is_read1 - read.is_read2]
        cand_to_genome.append(read.to_string())

        # check alignment against reference genome
        qual_filter = 1 if mapping_quality(blocks) <= mapping_quality(read.get_blocks()) + 5 else 0
        linear_filter = 1 if is_linear(read.cigartuples[0]) and is_linear(read.cigartuples[-1]) else 0
        align_filter = 1 if read.mapping_quality <= 10 or read.is_secondary or read.is_supplementary else 0
        if qual_filter or linear_filter or align_filter:
            fp_bsj.append((read.query_name, read.is_read1 - read.is_read2))

    if is_no_fsj:
        fsj_reads = []
    else:
        fsj_reads = []
        for circ_id, parser in CIRC[chrom].iteritems():
            # FSJ across start site
            for read in sam.fetch(region='{0}:{1}-{1}'.format(chrom, parser.start)):
                if read.is_unmapped or read.is_supplementary:
                    continue
                if read.mapping_quality <= 10:
                    continue
                if not read.get_overlap(parser.start - 1, parser.start + THRESHOLD - 1) >= THRESHOLD:
                    continue
                if is_mapped(read.cigartuples[0]) and is_mapped(read.cigartuples[-1]):
                    fsj_reads.append((read.query_name, read.is_read1 - read.is_read2, circ_id))

            for read in sam.fetch(region='{0}:{1}-{1}'.format(chrom, parser.end)):
                if read.is_unmapped or read.is_supplementary:
                    continue
                if read.mapping_quality <= 10:
                    continue
                if not read.get_overlap(parser.end - THRESHOLD, parser.end) >= THRESHOLD:
                    continue
                if is_mapped(read.cigartuples[0]) and is_mapped(read.cigartuples[-1]):
                    fsj_reads.append((read.query_name, read.is_read1 - read.is_read2, circ_id))

    sam.close()

    res = {'fp_bsj': fp_bsj, 'fsj_reads': fsj_reads, 'cand_to_genome': cand_to_genome}

    res_to_string = cPickle.dumps(res, 0)
    if sys.getsizeof(res_to_string) > 1024 * 1024 * 1024:
        pkl_file = "{}/{}.pkl".format(tmp_dir, chrom)
        cPickle.dump(res, open(pkl_file, "wb"), -1)
        return pkl_file

    return res

def mapping_quality(blocks):
    return sum([j - i for i, j in blocks])

def is_mapped(cigar_tuple):
    """
    Whether end of alignment segment is a mapped end

    Parameters
    -----
    cigar_tuple : tuple of cigar

    Returns
    -----
    int
        1 for linear end, 0 for ambiguous end

    """
    if cigar_tuple[0] == 0 or cigar_tuple[1] <= 10:
        return 1
    else:
        return 0

def is_linear(cigar_tuple):
    """
    Whether end of alignment segment is a linear end

    Parameters
    -----
    cigar_tuple : tuple of cigar

    Returns
    -----
    int
        1 for linear end, 0 for ambiguous end

    """
    if cigar_tuple[0] == 0 and cigar_tuple[1] >= 5:
        return 1
    else:
        return 0

def query_prefix(query_name):
    """
    Get pair id without mate id marker

    Paramters
    -----
    read : pysam.AlignedSegment

    Returns
    -----
    str
        mate id of segment

    """
    prefix_m = PREFIX.search(query_name)
    prefix = prefix_m.group(1) if prefix_m else query_name
    return prefix

def proc(log_file, thread, circ_file, hisat_bam, rnaser_file, reads, outdir, prefix, anchor, lib_type, is_no_fsj, bsj_file):
    """
    Build pseudo circular reference index and perform reads re-alignment
    Extract BSJ and FSJ reads from alignment results

    Returns
    -----
    str
        output file name

    """
    from utils import check_dir
    circ_dir = '{}/circ'.format(outdir)
    check_dir(circ_dir)

    circ_fasta = '{}/circ/{}_index.fa'.format(outdir, prefix)
    circ_info = load_bed(circ_file)
    if rnaser_file:
        LOGGER.info('Loading RNase R results')
        rnaser_exp, rnaser_stat = update_info(circ_info, rnaser_file)

    # extract fasta file for reads alignment
    generate_index(log_file, circ_info, circ_fasta)

    # hisat2-build index
    denovo_index = build_index(log_file, thread, circ_fasta, outdir, prefix)
    LOGGER.debug('De-novo index: {}'.format(denovo_index))

    # hisat2 de novo alignment for candidate reads
    denovo_bam = denovo_alignment(log_file, thread, reads, outdir, prefix)
    LOGGER.debug('De-novo bam: {}'.format(denovo_bam))

    # Find BSJ and FSJ informations
    cand_bsj = proc_denovo_bam(denovo_bam, thread, circ_info, anchor, lib_type)
    bsj_reads, fsj_reads = proc_genome_bam(hisat_bam, thread, circ_info, cand_bsj, anchor, circ_dir, is_no_fsj)
    if bsj_file is not None:
        LOGGER.info("Writing BSJ reads id to {}".format(bsj_file))
        output_bsj_ids(bsj_reads, bsj_file)

    total_reads, mapped_reads = bam_stat(hisat_bam)
    circ_reads = sum([len(bsj_reads[i]) for i in bsj_reads]) * 2
    sample_stat = (total_reads, mapped_reads, circ_reads)

    sample_exp = expression_level(circ_info, bsj_reads, fsj_reads)

    # circRNA annotation
    header = [
        'Sample: {}'.format(prefix),
        'Total_Reads: {}'.format(total_reads),
        'Mapped_Reads: {}'.format(mapped_reads),
        'Circular_Reads: {}'.format(circ_reads),
    ]
    out_file = '{}/{}.gtf'.format(outdir, prefix)

    if rnaser_file:
        import coeff
        tmp_header, circ_exp = coeff.correction(sample_exp, sample_stat, rnaser_exp, rnaser_stat)
        header += tmp_header
    else:
        circ_exp = sample_exp

    from version import __version__
    header += ['version: {}'.format(__version__), ]
    gtf_info = index_annotation(utils.GTF)
    format_output(circ_info, circ_exp, sample_stat, header, gtf_info, out_file)

    return out_file

def expression_level(circ_info, bsj_reads, fsj_reads):
    LOGGER.info('Merge bsj and fsj results')
    circ_exp = defaultdict(dict)
    bsj_ids = {}
    for chrom in circ_info:
        for circ_id in circ_info[chrom]:
            bsj = len(bsj_reads[circ_id]) if circ_id in bsj_reads else 0
            fsj = len(fsj_reads[circ_id]) if circ_id in fsj_reads else 0
            if bsj == 0 and fsj == 0:
                continue
            bsj_ids[circ_id] = bsj_reads[circ_id].keys()
            junc = 2.0 * bsj / (2.0 * bsj + fsj)
            circ_exp[circ_id] = {'bsj': bsj, 'fsj': fsj, 'ratio': junc}

    return circ_exp

def output_bsj_ids(bsj_reads, bsj_file):
    with open(bsj_file, 'w') as out:
        for circ_id in bsj_reads:
            for read_id in bsj_reads[circ_id]:
                out.write("{}\t{}\n".format(read_id, circ_id))
    return 0

def bam_stat(bam_file):
    """
    Stat of bam file

    Returns
    -----
    int
        number of total reads
    int
        number of mapped reads

    """
    sam = pysam.AlignmentFile(bam_file, 'rb')
    total = sam.count(read_callback=total_callback, until_eof=True)
    sam.close()

    sam = pysam.AlignmentFile(bam_file, 'rb')
    unmapped = sam.count(read_callback=unmapped_callback, until_eof=True)
    sam.close()

    return total, total - unmapped

def unmapped_callback(read):
    """
    callback for counting unmapped reads
    """
    return read.is_unmapped or read.mate_is_unmapped and not read.is_supplementary and not read.is_secondary

def total_callback(read):
    """
    callback for counting total reads
    """
    return not read.is_supplementary and not read.is_secondary

def format_output(circ_info, circ_exp, sample_stat, header, gtf_index, outfile):
    """
    Output bsj information of circRNA expression levels
    """
    LOGGER.info('Output circRNA expression values')

    with open(outfile, 'w') as out:
        for h in header:
            out.write('##' + h + '\n')
        for chrom in sorted(circ_info.keys(), key=by_chrom):
            for circ_id in sorted(circ_info[chrom].keys(), cmp=by_circ, key=lambda x:circ_info[chrom][x]):
                if circ_id not in circ_exp or circ_exp[circ_id]['bsj'] == 0:
                    continue
                parser = circ_info[chrom][circ_id]
                strand = parser.strand
                tmp_line = [
                    chrom,
                    'CIRIquant',
                    'circRNA',
                    parser.start,
                    parser.end,
                    '{:.4f}'.format(2 * 1000.0 * 1000.0 * circ_exp[circ_id]['bsj'] / sample_stat[1]),
                    strand,
                    '.',
                ]

                field = circRNA_attr(gtf_index, parser)
                tmp_attr = 'circ_id "{}"; circ_type "{}"; bsj {:.3f}; fsj {:.3f}; junc_ratio {:.3f};'.format(
                    circ_id,
                    field['circ_type'] if field else "Unknown",
                    circ_exp[circ_id]['bsj'],
                    circ_exp[circ_id]['fsj'],
                    circ_exp[circ_id]['ratio'],
                )
                for key in 'rnaser_bsj', 'rnaser_fsj':
                    if key in circ_exp[circ_id]:
                        tmp_attr += ' {} {:.3f};'.format(key, circ_exp[circ_id][key])

                for key in 'gene_id', 'gene_name', 'gene_type':
                    if key in field:
                        tmp_attr += ' {} "{}";'.format(key, field[key])

                tmp_line.append(tmp_attr)

                out.write('\t'.join([str(x) for x in tmp_line]) + '\n')
    return 1

def by_chrom(x):
    """
    Sort by chromosomes
    """
    chrom = x
    if x.startswith('chr'):
        chrom = chrom.strip('chr')
    try:
        chrom = int(chrom)
    except ValueError as e:
        pass
    return chrom

def by_circ(x, y):
    """
    Sort circRNAs by the start and end position
    """
    return x.end - y.end if x.start == y.start else x.start - y.start

class GTFParser(object):
    """
    Class for parsing annotation gtf
    """

    def __init__(self, content):
        self.chrom = content[0]
        self.source = content[1]
        self.type = content[2]
        self.start, self.end = int(content[3]), int(content[4])
        self.strand = content[6]
        self.attr_string = content[8]

    @property
    def attr(self):
        """
        Parsing attribute column in gtf file
        """
        field = {}
        for attr_values in [re.split(r'[\s=]+', i.strip()) for i in self.attr_string.split(';')[:-1]]:
            key, value = attr_values[0], attr_values[1:]
            field[key] = ' '.join(value).strip('"')
        return field

def index_annotation(gtf):
    """
    Generate binned index for element in gtf
    """

    LOGGER.info('Loading annotation gtf ..')
    gtf_index = defaultdict(dict)
    with open(gtf, 'r') as f:
        for line in f:
            if line.startswith('#'):
                continue
            content = line.rstrip().split('\t')
            # only include gene and exon feature for now
            if content[2] not in ['gene', 'exon']:
                continue
            parser = GTFParser(content)
            # if 'gene_biotype' in parser.attr and parser.attr['gene_biotype'] in ['lincRNA', 'pseudogene']:
            #     continue
            # if 'gene_type' in parser.attr and parser.attr['gene_type'] in ['lincRNA', 'pseudogene']:
            #     continue

            start_div, end_div = parser.start / 500, parser.end / 500
            for i in xrange(start_div, end_div + 1):
                gtf_index[parser.chrom].setdefault(i, []).append(parser)
    return gtf_index

def circRNA_attr(gtf_index, circ):
    """
    annotate circRNA information
    """
    if circ.chrom not in gtf_index:
        LOGGER.warn('chrom of contig "{}" not in annotation gtf, please check'.format(circ.chrom))
        return {}
    start_div, end_div = circ.start / 500, circ.end / 500

    host_gene = {}
    start_element = defaultdict(list)
    end_element = defaultdict(list)

    for x in xrange(start_div, end_div + 1):
        if x not in gtf_index[circ.chrom]:
            continue
        for element in gtf_index[circ.chrom][x]:
            # start site
            if element.start <= circ.start <= element.end and element.strand == circ.strand:
                start_element[element.type].append(element)
            # end site
            if element.start <= circ.end <= element.end and element.strand == circ.strand:
                end_element[element.type].append(element)
            # if element.type != 'gene':
            #     continue
            if element.end < circ.start or circ.end < element.start:
                continue
            if element.attr['gene_id'] not in host_gene:
                host_gene[element.attr['gene_id']] = element

    circ_type = {}
    forward_host_gene = []
    antisense_host_gene = []

    if len(host_gene) > 0:
        for gene_id in host_gene:
            if host_gene[gene_id].strand == circ.strand:
                forward_host_gene.append(host_gene[gene_id])
                if 'exon' in start_element and 'exon' in end_element:
                    circ_type['exon'] = 1
                else:
                    circ_type['intron'] = 1
            else:
                antisense_host_gene.append(host_gene[gene_id])
                circ_type['antisense'] = 1
    else:
        circ_type['intergenic'] = 1

    if len(forward_host_gene) > 1:
        circ_type['gene_intergenic'] = 1

    field = {}
    if 'exon' in circ_type:
        field['circ_type'] = 'exon'
    elif 'intron' in circ_type:
        field['circ_type'] = 'intron'
    # elif 'gene_intergenic' in circ_type:
    #     field['circ_type'] = 'gene_intergenic'
    elif 'antisense' in circ_type:
        field['circ_type'] = 'antisense'
    else:
        field['circ_type'] = 'intergenic'

    if len(forward_host_gene) == 1:
        if 'gene_id' in forward_host_gene[0].attr:
            field['gene_id'] = forward_host_gene[0].attr['gene_id']
        if 'gene_name' in forward_host_gene[0].attr:
            field['gene_name'] = forward_host_gene[0].attr['gene_name']
        if 'gene_type' in forward_host_gene[0].attr:
            field['gene_type'] = forward_host_gene[0].attr['gene_type']
        elif 'gene_biotype' in forward_host_gene[0].attr:
            field['gene_type'] = forward_host_gene[0].attr['gene_biotype']
        else:
            pass
    elif len(forward_host_gene) > 1:
        tmp_gene_id = []
        tmp_gene_name = []
        tmp_gene_type = []
        for x in forward_host_gene:
            if 'gene_id' in x.attr:
                tmp_gene_id.append(x.attr['gene_id'])
            if 'gene_name' in x.attr:
                tmp_gene_name.append(x.attr['gene_name'])
            if 'gene_type' in x.attr:
                tmp_gene_type.append(x.attr['gene_type'])
            elif 'gene_biotype' in x.attr:
                tmp_gene_type.append(x.attr['gene_biotype'])
            else:
                pass
        if len(tmp_gene_id) > 0:
            field['gene_id'] = ','.join(tmp_gene_id)
        if len(tmp_gene_name) > 0:
            field['gene_name'] = ','.join(tmp_gene_name)
        if len(tmp_gene_type) > 0:
            field['gene_type'] = ','.join(tmp_gene_type)
    elif field['circ_type'] == 'antisense' and len(antisense_host_gene) > 0:
        tmp_gene_id = []
        tmp_gene_name = []
        tmp_gene_type = []
        for x in antisense_host_gene:
            if 'gene_id' in x.attr:
                tmp_gene_id.append(x.attr['gene_id'])
            if 'gene_name' in x.attr:
                tmp_gene_name.append(x.attr['gene_name'])
            if 'gene_type' in x.attr:
                tmp_gene_type.append(x.attr['gene_type'])
            elif 'gene_biotype' in x.attr:
                tmp_gene_type.append(x.attr['gene_biotype'])
            else:
                pass
        if len(tmp_gene_id) > 0:
            field['gene_id'] = ','.join(tmp_gene_id)
        if len(tmp_gene_name) > 0:
            field['gene_name'] = ','.join(tmp_gene_name)
        if len(tmp_gene_type) > 0:
            field['gene_type'] = ','.join(tmp_gene_type)
    else:
        pass

    return field


from gettext import gettext as _
from typing import cast

from telegram import (
    CallbackQuery,
    InlineKeyboardButton,
    InlineKeyboardMarkup,
    LabeledPrice,
    Message,
    PreCheckoutQuery,
    ReplyKeyboardRemove,
    Update,
)
from telegram.ext import ContextTypes

from pdf_bot.errors import CallbackQueryDataTypeError
from pdf_bot.language import LanguageService
from pdf_bot.telegram_internal import TelegramService

from .models import PaymentData

class PaymentService:
    _INVOICE_PAYLOAD = "invoice_payload"
    _CURRENCY = "USD"
    _PAYMENT_MESSAGE = _("{message} {emoji} (${value})")
    _KEYBOARD_SIZE = 2

    _PAYMENT_DATA_LIST = (
        PaymentData(label=_("Say Thanks"), emoji="😁", value=1),
        PaymentData(label=_("Coffee"), emoji="☕", value=3),
        PaymentData(label=_("Beer"), emoji="🍺", value=5),
        PaymentData(label=_("Meal"), emoji="🍲", value=10),
    )

    def __init__(
        self,
        language_service: LanguageService,
        telegram_service: TelegramService,
        stripe_token: str,
    ) -> None:
        self.language_service = language_service
        self.telegram_service = telegram_service
        self.stripe_token = stripe_token

    async def send_support_options(
        self, update: Update, context: ContextTypes.DEFAULT_TYPE
    ) -> None:
        query: CallbackQuery | None = update.callback_query

        # This method is used by both command and callback query handlers, so we need to
        # check if query is `None` here
        if query is not None:
            await query.answer()

        _ = self.language_service.set_app_language(update, context)
        msg = cast(Message, update.effective_message)
        reply_markup = self._get_support_options_markup(update, context)
        await msg.reply_text(_("Select how you want to support PDF Bot"), reply_markup=reply_markup)

    async def send_invoice(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
        query: CallbackQuery = cast(CallbackQuery, update.callback_query)
        await self.telegram_service.answer_query_and_drop_data(context, query)
        data: str | PaymentData | None = query.data

        if not isinstance(data, PaymentData):
            raise CallbackQueryDataTypeError(data)

        _ = self.language_service.set_app_language(update, context)
        msg = cast(Message, update.effective_message)
        prices = [LabeledPrice(data.label, data.value * 100)]

        await msg.reply_invoice(
            title=_("Support PDF Bot"),
            description=_("Say thanks to PDF Bot and help keep it running"),
            payload=self._INVOICE_PAYLOAD,
            provider_token=self.stripe_token,
            currency=self._CURRENCY,
            prices=prices,
            max_tip_amount=1000,
            suggested_tip_amounts=[100, 300, 500, 1000],
        )

    async def precheckout_check(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
        _ = self.language_service.set_app_language(update, context)
        query = cast(PreCheckoutQuery, update.pre_checkout_query)

        if query.invoice_payload != self._INVOICE_PAYLOAD:
            await query.answer(ok=False, error_message=_("Something went wrong, try again"))
        else:
            await query.answer(ok=True)

    async def successful_payment(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
        _ = self.language_service.set_app_language(update, context)
        msg = cast(Message, update.effective_message)
        await msg.reply_text(_("Thank you for your support!"), reply_markup=ReplyKeyboardRemove())

    def _get_support_options_markup(
        self, update: Update, context: ContextTypes.DEFAULT_TYPE
    ) -> InlineKeyboardMarkup:
        _ = self.language_service.set_app_language(update, context)
        keyboard = [
            [
                InlineKeyboardButton(
                    _(self._PAYMENT_MESSAGE).format(
                        message=_(data.label), emoji=data.emoji, value=data.value
                    ),
                    callback_data=data,
                )
                for data in self._PAYMENT_DATA_LIST[i : i + self._KEYBOARD_SIZE]
            ]
            for i in range(0, len(self._PAYMENT_DATA_LIST), self._KEYBOARD_SIZE)
        ]
        keyboard.append(
            [InlineKeyboardButton(_("Help translate PDF Bot"), "https://crwd.in/telegram-pdf-bot")]
        )

        return InlineKeyboardMarkup(keyboard)


import jieba
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from functools import partial

# the stopwords #
def stopwords(root):
    f = open(root, "r", encoding = 'utf8')
    lines = f.readlines()
    stopwords = [word.strip() for word in lines]
    stopwords.append(' ')
    f.close()
    return stopwords

# the training data #

def init_data(root, stopwords, maxlen):
    f = open(root, 'r', encoding = 'utf8')
    lines = f.readlines()
    dataset = []
    pad_lst = ['<pad>']
    for line in lines:
        packed = line.strip().split("\t")
        packed[0] = list(jieba.cut(packed[0]))
        packed[0] = [word for word in packed[0] if word not in stopwords and ' ' not in word]
        if len(packed[0]) < maxlen:
            packed[0] += pad_lst * (maxlen - len(packed[0]))
        else:
            packed[0] = packed[0][:maxlen]
        dataset.append({'text':packed[0], 'label':packed[1]})
    return dataset

# vocab #
def vocabulary(train_data, dev_data, test_data):
    vocab = [word for data in train_data for word in data['text']]
    vocab += [word for data in dev_data for word in data['text']]
    vocab += [word for data in test_data for word in data['text']]
    vocab.append('<pad>')
    return set(vocab)

def word2idx(vocab):
    idx2word = sorted(list(vocab))
    word2idx = {}
    for num, word in enumerate(idx2word):
        word2idx[word] = num
    return idx2word, word2idx

def make_batch(data, word2idx):
    train_sentences = []
    train_labels = []
    for item in data:
        item_text = [word2idx[word] for word in item['text']]
        train_sentences.append(item_text)
        train_labels.append(eval(item['label']))
    return list(zip(train_sentences, train_labels))

def custom_collate_fn(batch, word2idx, device):
    x, y = zip(*batch)

    pad_token_ix = word2idx['<pad>']
    x = [torch.LongTensor(x_i, device = device) for x_i in x]
    x_padded = nn.utils.rnn.pad_sequence(x, batch_first = True, padding_value = pad_token_ix)
    y = torch.LongTensor(y, device = device)

    return x_padded, y