<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package federation

import (
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

// ServerAddressByClientCIDR helps the client to determine the server address that they should use, depending on the clientCIDR that they match.
type ServerAddressByClientCIDR struct {
	// The CIDR with which clients can match their IP to figure out the server address that they should use.
	ClientCIDR string `json:"clientCIDR" protobuf:"bytes,1,opt,name=clientCIDR"`
	// Address of this server, suitable for a client that matches the above CIDR.
	// This can be a hostname, hostname:port, IP or IP:port.
	ServerAddress string `json:"serverAddress" protobuf:"bytes,2,opt,name=serverAddress"`
}

// ClusterSpec describes the attributes of a kubernetes cluster.
type ClusterSpec struct {
	// A map of client CIDR to server address.
	// This is to help clients reach servers in the most network-efficient way possible.
	// Clients can use the appropriate server address as per the CIDR that they match.
	// In case of multiple matches, clients should use the longest matching CIDR.
	ServerAddressByClientCIDRs []ServerAddressByClientCIDR `json:"serverAddressByClientCIDRs" patchStrategy:"merge" patchMergeKey:"clientCIDR"`
	// Name of the secret containing kubeconfig to access this cluster.
	// The secret is read from the kubernetes cluster that is hosting federation control plane.
	// Admin needs to ensure that the required secret exists. Secret should be in the same namespace where federation control plane is hosted and it should have kubeconfig in its data with key "kubeconfig".
	// This will later be changed to a reference to secret in federation control plane when the federation control plane supports secrets.
	// This can be left empty if the cluster allows insecure access.
	// +optional
	SecretRef *api.LocalObjectReference `json:"secretRef,omitempty"`
}

type ClusterConditionType string

// These are valid conditions of a cluster.
const (
	// ClusterReady means the cluster is ready to accept workloads.
	ClusterReady ClusterConditionType = "Ready"
	// ClusterOffline means the cluster is temporarily down or not reachable
	ClusterOffline ClusterConditionType = "Offline"
)

// ClusterCondition describes current state of a cluster.
type ClusterCondition struct {
	// Type of cluster condition, Complete or Failed.
	Type ClusterConditionType `json:"type"`
	// Status of the condition, one of True, False, Unknown.
	Status api.ConditionStatus `json:"status"`
	// Last time the condition was checked.
	// +optional
	LastProbeTime unversioned.Time `json:"lastProbeTime,omitempty"`
	// Last time the condition transit from one status to another.
	// +optional
	LastTransitionTime unversioned.Time `json:"lastTransitionTime,omitempty"`
	// (brief) reason for the condition's last transition.
	// +optional
	Reason string `json:"reason,omitempty"`
	// Human readable message indicating details about last transition.
	// +optional
	Message string `json:"message,omitempty"`
}

// ClusterStatus is information about the current status of a cluster updated by cluster controller peridocally.
type ClusterStatus struct {
	// Conditions is an array of current cluster conditions.
	// +optional
	Conditions []ClusterCondition `json:"conditions,omitempty"`
	// Zones is the list of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.
	// These will always be in the same region.
	// +optional
	Zones []string `json:"zones,omitempty"`
	// Region is the name of the region in which all of the nodes in the cluster exist.  e.g. 'us-east1'.
	// +optional
	Region string `json:"region,omitempty"`
}

// +genclient=true
// +nonNamespaced=true

// Information about a registered cluster in a federated kubernetes setup. Clusters are not namespaced and have unique names in the federation.
type Cluster struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	api.ObjectMeta `json:"metadata,omitempty"`

	// Spec defines the behavior of the Cluster.
	// +optional
	Spec ClusterSpec `json:"spec,omitempty"`
	// Status describes the current status of a Cluster
	// +optional
	Status ClusterStatus `json:"status,omitempty"`
}

// A list of all the kubernetes clusters registered to the federation
type ClusterList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard list metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#types-kinds
	// +optional
	unversioned.ListMeta `json:"metadata,omitempty"`

	// List of Cluster objects.
	Items []Cluster `json:"items"`
}

// Temporary/alpha structures to support custom replica assignments within FederatedReplicaSet.

// A set of preferences that can be added to federated version of ReplicaSet as a json-serialized annotation.
// The preferences allow the user to express in which culsters he wants to put his replicas within the
// mentiond FederatedReplicaSet.
type FederatedReplicaSetPreferences struct {
	// If set to true then already scheduled and running replicas may be moved to other clusters to
	// in order to bring cluster replicasets towards a desired state. Otherwise, if set to false,
	// up and running replicas will not be moved.
	// +optional
	Rebalance bool `json:"rebalance,omitempty"`

	// A mapping between cluser names and preferences regarding local replicasets in these clusters.
	// "*" (if provided) applies to all clusters if an explicit mapping is not provided. If there is no
	// "*" that clusters without explicit preferences should not have any replicas scheduled.
	// +optional
	Clusters map[string]ClusterReplicaSetPreferences `json:"clusters,omitempty"`
}

// Preferences regarding number of replicas assigned to a cluster replicaset within a federated replicaset.
type ClusterReplicaSetPreferences struct {
	// Minimum number of replicas that should be assigned to this Local ReplicaSet. 0 by default.
	// +optional
	MinReplicas int64 `json:"minReplicas,omitempty"`

	// Maximum number of replicas that should be assigned to this Local ReplicaSet. Unbounded if no value provided (default).
	// +optional
	MaxReplicas *int64 `json:"maxReplicas,omitempty"`

	// A number expressing the preference to put an additional replica to this LocalReplicaSet. 0 by default.
	Weight int64
}
<commit_msg>Update types.go<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package federation

import (
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

// ServerAddressByClientCIDR helps the client to determine the server address that they should use, depending on the clientCIDR that they match.
type ServerAddressByClientCIDR struct {
	// The CIDR with which clients can match their IP to figure out the server address that they should use.
	ClientCIDR string `json:"clientCIDR" protobuf:"bytes,1,opt,name=clientCIDR"`
	// Address of this server, suitable for a client that matches the above CIDR.
	// This can be a hostname, hostname:port, IP or IP:port.
	ServerAddress string `json:"serverAddress" protobuf:"bytes,2,opt,name=serverAddress"`
}

// ClusterSpec describes the attributes of a kubernetes cluster.
type ClusterSpec struct {
	// A map of client CIDR to server address.
	// This is to help clients reach servers in the most network-efficient way possible.
	// Clients can use the appropriate server address as per the CIDR that they match.
	// In case of multiple matches, clients should use the longest matching CIDR.
	ServerAddressByClientCIDRs []ServerAddressByClientCIDR `json:"serverAddressByClientCIDRs" patchStrategy:"merge" patchMergeKey:"clientCIDR"`
	// Name of the secret containing kubeconfig to access this cluster.
	// The secret is read from the kubernetes cluster that is hosting federation control plane.
	// Admin needs to ensure that the required secret exists. Secret should be in the same namespace where federation control plane is hosted and it should have kubeconfig in its data with key "kubeconfig".
	// This will later be changed to a reference to secret in federation control plane when the federation control plane supports secrets.
	// This can be left empty if the cluster allows insecure access.
	// +optional
	SecretRef *api.LocalObjectReference `json:"secretRef,omitempty"`
}

type ClusterConditionType string

// These are valid conditions of a cluster.
const (
	// ClusterReady means the cluster is ready to accept workloads.
	ClusterReady ClusterConditionType = "Ready"
	// ClusterOffline means the cluster is temporarily down or not reachable
	ClusterOffline ClusterConditionType = "Offline"
)

// ClusterCondition describes current state of a cluster.
type ClusterCondition struct {
	// Type of cluster condition, Complete or Failed.
	Type ClusterConditionType `json:"type"`
	// Status of the condition, one of True, False, Unknown.
	Status api.ConditionStatus `json:"status"`
	// Last time the condition was checked.
	// +optional
	LastProbeTime unversioned.Time `json:"lastProbeTime,omitempty"`
	// Last time the condition transit from one status to another.
	// +optional
	LastTransitionTime unversioned.Time `json:"lastTransitionTime,omitempty"`
	// (brief) reason for the condition's last transition.
	// +optional
	Reason string `json:"reason,omitempty"`
	// Human readable message indicating details about last transition.
	// +optional
	Message string `json:"message,omitempty"`
}

// ClusterStatus is information about the current status of a cluster updated by cluster controller peridocally.
type ClusterStatus struct {
	// Conditions is an array of current cluster conditions.
	// +optional
	Conditions []ClusterCondition `json:"conditions,omitempty"`
	// Zones is the list of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.
	// These will always be in the same region.
	// +optional
	Zones []string `json:"zones,omitempty"`
	// Region is the name of the region in which all of the nodes in the cluster exist.  e.g. 'us-east1'.
	// +optional
	Region string `json:"region,omitempty"`
}

// +genclient=true
// +nonNamespaced=true

// Information about a registered cluster in a federated kubernetes setup. Clusters are not namespaced and have unique names in the federation.
type Cluster struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	api.ObjectMeta `json:"metadata,omitempty"`

	// Spec defines the behavior of the Cluster.
	// +optional
	Spec ClusterSpec `json:"spec,omitempty"`
	// Status describes the current status of a Cluster
	// +optional
	Status ClusterStatus `json:"status,omitempty"`
}

// A list of all the kubernetes clusters registered to the federation
type ClusterList struct {
	unversioned.TypeMeta `json:",inline"`
	// Standard list metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#types-kinds
	// +optional
	unversioned.ListMeta `json:"metadata,omitempty"`

	// List of Cluster objects.
	Items []Cluster `json:"items"`
}

// Temporary/alpha structures to support custom replica assignments within FederatedReplicaSet.

// A set of preferences that can be added to federated version of ReplicaSet as a json-serialized annotation.
// The preferences allow the user to express in which culsters he wants to put his replicas within the
// mentiond FederatedReplicaSet.
type FederatedReplicaSetPreferences struct {
	// If set to true then already scheduled and running replicas may be moved to other clusters to
	// in order to bring cluster replicasets towards a desired state. Otherwise, if set to false,
	// up and running replicas will not be moved.
	// +optional
	Rebalance bool `json:"rebalance,omitempty"`

	// A mapping between cluster names and preferences regarding local ReplicaSet in these clusters.
	// "*" (if provided) applies to all clusters if an explicit mapping is not provided. If there is no
	// "*" that clusters without explicit preferences should not have any replicas scheduled.
	// +optional
	Clusters map[string]ClusterReplicaSetPreferences `json:"clusters,omitempty"`
}

// Preferences regarding number of replicas assigned to a cluster replicaset within a federated replicaset.
type ClusterReplicaSetPreferences struct {
	// Minimum number of replicas that should be assigned to this Local ReplicaSet. 0 by default.
	// +optional
	MinReplicas int64 `json:"minReplicas,omitempty"`

	// Maximum number of replicas that should be assigned to this Local ReplicaSet. Unbounded if no value provided (default).
	// +optional
	MaxReplicas *int64 `json:"maxReplicas,omitempty"`

	// A number expressing the preference to put an additional replica to this LocalReplicaSet. 0 by default.
	Weight int64
}
<|endoftext|>


<commit_before>// Package router provides a simple router to match URL's
// with Actions and support for parameters and options.
// TODO options
package router

import (
	"errors"
	"strings"
)

// Router interface is a generic specification of the router
type RouterInterface interface {
	GetRoute(url string) (*Route, error)
}

// Route struct contains all the information for the route
// including the Controller and the Method to call
type Route struct {
	Pattern    string
	Action     string
	Method     string
}

// Router struct stores all the routes configured using
// a simple map of the url to the respective Route struct
// TODO improve mapping
type Router struct {
	Routes map[string]*Route
}

// Router Handler method, receives a url and return the
// associated route
func (router *Router) GetRoute(url string) (*Route, error) {

	var matchedRoute *Route
	foundRoute := false
	for _, route := range router.Routes {
		if matchRoute(route.Pattern, url) {
			matchedRoute = route
			foundRoute = true
		}
	}
	var err error
	if !foundRoute {
		err = errors.New("Route not found:" + url)
	}

	return matchedRoute, err
}

// Verifies if a URL matches a given pattern
// Only supports static routes and simple parameters declared with an ':'
// before the name
func matchRoute(urlPattern string, urlRecieved string) bool {
	splitUrlPattern := strings.Split(strings.Trim(urlPattern, "/"), "/")
	splitUrlRecieved := strings.Split(strings.Trim(urlRecieved, "/"), "/")

	if len(splitUrlRecieved) > len(splitUrlPattern) {
		return false
	}

	for index, urlPatternElement := range splitUrlPattern {
		if urlPatternElement[:1] == ":" {
			continue
		}
		if len(splitUrlRecieved) > index {
			if urlPatternElement != splitUrlRecieved[index] {
				return false
			}
		} else {
			return false
		}
	}

	return true
}
}
<commit_msg>added url parameter extraction<commit_after>// Package router provides a simple router to match URL's
// with Actions and support for parameters and options.
// TODO options
package router

import (
	"errors"
	"strings"
)

// Router interface is a generic specification of the router
type RouterInterface interface {
	GetRoute(url string) (*Route, error)
}

// Route struct contains all the information for the route
// including the Controller and the Method to call
type Route struct {
	Pattern    string
	Action     string
	Method     string
	parameters map[string]string
}

// Router struct stores all the routes configured using
// a simple map of the url to the respective Route struct
// TODO improve mapping
type Router struct {
	Routes map[string]*Route
}

// Router Handler method, receives a url and return the
// associated route
func (router *Router) GetRoute(url string) (*Route, error) {

	var matchedRoute *Route
	foundRoute := false
	for _, route := range router.Routes {
		if matchRoute(route.Pattern, url) {
			matchedRoute = route
			foundRoute = true
			route.parameters = extractParamFromUrl(route.Pattern, url)
		}
	}
	var err error
	if !foundRoute {
		err = errors.New("Route not found:" + url)
	}

	return matchedRoute, err
}

// Verifies if a URL matches a given pattern
// Only supports static routes and simple parameters declared with an ':'
// before the name
func matchRoute(urlPattern string, urlRecieved string) bool {
	splitUrlPattern := strings.Split(strings.Trim(urlPattern, "/"), "/")
	splitUrlRecieved := strings.Split(strings.Trim(urlRecieved, "/"), "/")

	if len(splitUrlRecieved) > len(splitUrlPattern) {
		return false
	}

	for index, urlPatternElement := range splitUrlPattern {
		if urlPatternElement[:1] == ":" {
			continue
		}
		if len(splitUrlRecieved) > index {
			if urlPatternElement != splitUrlRecieved[index] {
				return false
			}
		} else {
			return false
		}
	}

	return true
}

// Extracts the parameters from an URL matching a given pattern
// Assumes that the URL has already matched the pattern
func extractParamFromUrl(urlPattern string, urlRecieved string) map[string]string {
	splitUrlPattern := strings.Split(strings.Trim(urlPattern, "/"), "/")
	splitUrlRecieved := strings.Split(strings.Trim(urlRecieved, "/"), "/")

	params := make(map[string]string)

	var paramValue string
	var paramKey string
	for index, urlPatternElement := range splitUrlPattern {
		if urlPatternElement[:1] == ":" {
			paramKey = urlPatternElement[1:len(urlPatternElement)]
			if len(splitUrlRecieved) > index {
				paramValue = splitUrlRecieved[index]
			} else {
				paramValue = ""
			}
			params[paramKey] = paramValue
		}
	}

	return params
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"os"
	"runtime"
	"time"

	keybase1 "github.com/keybase/client/go/protocol"
)

const (
	DevelServerURI      = "http://localhost:3000"
	StagingServerURI    = "https://stage0.keybase.io"
	ProductionServerURI = "https://keybase.io"
	TorServerURI        = "http://fncuwbiisyh6ak3i.onion"
)

type RunMode string

var TorProxy = "localhost:9050"

const (
	DevelRunMode      RunMode = "devel"
	StagingRunMode            = "staging"
	ProductionRunMode         = "prod"
	RunModeError              = "error"
	NoRunMode                 = ""
)

var RunModes = []RunMode{DevelRunMode, StagingRunMode, ProductionRunMode}

var ServerLookup = map[RunMode]string{
	DevelRunMode:      DevelServerURI,
	StagingRunMode:    StagingServerURI,
	ProductionRunMode: ProductionServerURI,
}

const (
	ConfigFile  = "config.json"
	SessionFile = "session.json"
	DBFile      = "keybase.leveldb"
	SocketFile  = "keybased.sock"
	PIDFile     = "keybased.pid"

	SecretKeyringTemplate = "secretkeys.%u.mpack"

	APIVersion       = "1.0"
	APIURIPathPrefix = "/_/api/" + APIVersion
	DaemonPort       = 40933
	GoClientID       = "keybase.io go client"
	IdentifyAs       = GoClientID + " v" + Version + " " + runtime.GOOS
)

var UserAgent = "Keybase/" + Version + " (" + runtime.Version() + " on " + runtime.GOOS + ")"

const (
	PermFile          os.FileMode = 0600
	PermDir           os.FileMode = 0700
	UmaskablePermFile os.FileMode = 0666
)

const (
	UserCacheMaxAge      = 5 * time.Minute
	PGPFingerprintHexLen = 40

	ProofCacheSize      = 0x1000
	ProofCacheLongDur   = 6 * time.Hour
	ProofCacheMediumDur = 30 * time.Minute
	ProofCacheShortDur  = 1 * time.Minute

	SigShortIDBytes = 27
)

var MerkleProdKIDs = []string{
	"010159baae6c7d43c66adf8fb7bb2b8b4cbe408c062cfc369e693ccb18f85631dbcd0a",
}
var MerkleTestKIDs = []string{
	"0101be58b6c82db64f6ccabb05088db443c69f87d5d48857d709ed6f73948dabe67d0a",
}
var MerkleStagingKIDs = []string{
	"0101bed85ce72cc315828367c28b41af585b6b7d95646a62ca829691d70f49184fa70a",
}

const (
	KeybaseKIDV1       = 1 // Uses SHA-256
	KeybaseSignatureV1 = 1
	OneYearInSeconds   = 24 * 60 * 60 * 365

	SigExpireIn       = OneYearInSeconds * 16 // 16 years
	NaclEdDSAExpireIn = OneYearInSeconds * 16 // 16 years
	NaclDHExpireIn    = OneYearInSeconds * 16 // 16 years
	KeyExpireIn       = OneYearInSeconds * 16 // 16 years
	SubkeyExpireIn    = OneYearInSeconds * 16 // 16 years
	AuthExpireIn      = OneYearInSeconds      // 1 year
)

// Status codes.  This list should match keybase/lib/constants.iced.
const (
	SCOk                     = 0
	SCLoginRequired          = 201
	SCBadSession             = 202
	SCBadLoginPassword       = 204
	SCNotFound               = 205
	SCGeneric                = 218
	SCAlreadyLoggedIn        = 235
	SCCanceled               = 237
	SCReloginRequired        = 274
	SCBadSignupUsernameTaken = 701
	SCKeyNotFound            = 901
	SCKeyInUse               = 907
	SCKeyBadGen              = 913
	SCKeyNoSecret            = 914
	SCKeyBadUIDs             = 915
	SCKeyNoActive            = 916
	SCKeyNoSig               = 917
	SCKeyBadSig              = 918
	SCKeyBadEldest           = 919
	SCKeyNoEldest            = 920
	SCKeyDuplicateUpdate     = 921
	SCSibkeyAlreadyExists    = 922
	SCDecryptionKeyNotFound  = 924
	SCBadTrackSession        = 1301
	SCDeviceNotFound         = 1409
	SCDeviceMismatch         = 1410
	SCDeviceRequired         = 1411
	SCStreamExists           = 1501
	SCStreamNotFound         = 1502
	SCStreamWrongKind        = 1503
	SCStreamEOF              = 1504
	SCAPINetworkError        = 1601
	SCTimeout                = 1602
	SCProofError             = 1701
	SCIdentificationExpired  = 1702
	SCSelfNotFound           = 1703
	SCBadKexPhrase           = 1704
	SCNoUIDelegation         = 1705
)

const (
	IDSuffixKID = 0x0a
)

const (
	MerkleTreeNode = 1
	MerkleTreeLeaf = 2
)

type LinkType string
type DelegationType LinkType

const (
	AuthenticationType    LinkType = "auth"
	CryptocurrencyType             = "cryptocurrency"
	RevokeType                     = "revoke"
	TrackType                      = "track"
	UntrackType                    = "untrack"
	UpdatePassphraseType           = "update_passphrase_hash"
	WebServiceBindingType          = "web_service_binding"

	EldestType    DelegationType = "eldest"
	PGPUpdateType                = "pgp_update"
	SibkeyType                   = "sibkey"
	SubkeyType                   = "subkey"
)

const (
	SigTypeNone           = 0
	SigTypeSelfSig        = 1
	SigTypeRemoteProof    = 2
	SigTypeTrack          = 3
	SigTypeUntrack        = 4
	SigTypeRevoke         = 5
	SigTypeCryptocurrency = 6
	SigTypeAnnouncement   = 7
)

type KeyType int

const (
	KeyTypeNone                  KeyType = 0
	KeyTypeOpenPGPPublic                 = 1
	KeyTypeP3skbPrivate                  = 2
	KeyTypeKbNaclEddsa                   = 3
	KeyTypeKbNaclDH                      = 4
	KeyTypeKbNaclEddsaServerHalf         = 5
	KeyTypeKbNaclDHServerHalf            = 6
)

const (
	DeviceStatusNone    = 0
	DeviceStatusActive  = 1
	DeviceStatusDefunct = 2
)

// these strings need to match the keys in
// keybase/lib_public/public_constants.iced ->
// public_constants.device.type
const (
	DeviceTypeDesktop = "desktop"
	DeviceTypeMobile  = "mobile"
	DeviceTypePaper   = "backup"
)

const DownloadURL = "https://keybase.io/download"

var PGPVersion = "Keybase Go " + Version + " (" + runtime.GOOS + ")"

var PGPArmorHeaders = map[string]string{
	"Version": PGPVersion,
	"Comment": DownloadURL,
}

var RemoteServiceTypes = map[string]keybase1.ProofType{
	"keybase":    keybase1.ProofType_KEYBASE,
	"twitter":    keybase1.ProofType_TWITTER,
	"github":     keybase1.ProofType_GITHUB,
	"reddit":     keybase1.ProofType_REDDIT,
	"coinbase":   keybase1.ProofType_COINBASE,
	"hackernews": keybase1.ProofType_HACKERNEWS,
	"https":      keybase1.ProofType_GENERIC_WEB_SITE,
	"http":       keybase1.ProofType_GENERIC_WEB_SITE,
	"dns":        keybase1.ProofType_DNS,
	"rooter":     keybase1.ProofType_ROOTER,
}

var RemoteServiceOrder = []keybase1.ProofType{
	keybase1.ProofType_KEYBASE,
	keybase1.ProofType_TWITTER,
	keybase1.ProofType_GITHUB,
	keybase1.ProofType_REDDIT,
	keybase1.ProofType_COINBASE,
	keybase1.ProofType_HACKERNEWS,
	keybase1.ProofType_GENERIC_WEB_SITE,
	keybase1.ProofType_ROOTER,
}

const CanonicalHost = "keybase.io"

const (
	HTTPDefaultTimeout = 60 * time.Second
	HTTPPollMaximum    = 5 * time.Second
)

// Packet tags for OpenPGP and also Keybase packets
const (
	KeybasePacketV1 = 1
	TagP3skb        = 513
	TagSignature    = 514
	TagEncryption   = 515
)

const (
	KIDPGPBase    AlgoType = 0x00
	KIDPGPRsa              = 0x1
	KIDPGPElgamal          = 0x10
	KIDPGPDsa              = 0x11
	KIDPGPEcdh             = 0x12
	KIDPGPEcdsa            = 0x13
	KIDNaclEddsa           = 0x20
	KIDNaclDH              = 0x21
)

// OpenPGP hash IDs, taken from http://tools.ietf.org/html/rfc4880#section-9.4
const (
	HashPGPMd5       = 1
	HashPGPSha1      = 2
	HashPGPRipemd160 = 3
	HashPGPSha256    = 8
	HashPGPSha384    = 9
	HashPGPSha512    = 10
	HashPGPSha224    = 11
)

const (
	SigKbEddsa = KIDNaclEddsa
)

const (
	ServerUpdateLag = time.Minute
)

// key_revocation_types
const (
	RevSimpleDelete = 0
	RevFull         = 1
	RevDated        = 2
)

type KeyStatus int

const (
	KeyUncancelled KeyStatus = iota
	KeyRevoked
	KeyDeleted
	KeySuperseded
)

type KeyRole int

const (
	DLGNone KeyRole = iota
	DLGSibkey
	DLGSubkey
)

const (
	KexScryptCost       = 32768
	KexScryptR          = 8
	KexScryptP          = 1
	KexScryptKeylen     = 32
	KexSessionIDEntropy = 65 // kex doc specifies 65 bits of entropy
)

const (
	Kex2PhraseEntropy = 88
	Kex2ScryptCost    = 1 << 17
	Kex2ScryptR       = 8
	Kex2ScryptP       = 1
	Kex2ScryptKeylen  = 32
)

const (
	PaperKeyScryptCost    = 32768
	PaperKeyScryptR       = 8
	PaperKeyScryptP       = 1
	PaperKeyScryptKeylen  = 128
	PaperKeySecretEntropy = 117
	PaperKeyIDBits        = 22
	PaperKeyVersionBits   = 4
	PaperKeyVersion       = 0
)

const UserSummaryLimit = 500 // max number of user summaries in one request

const MinPassphraseLength = 12

type KexRole int

const (
	KexRoleProvisioner KexRole = iota
	KexRoleProvisionee
)
<commit_msg>Add IdentifySourceKBFS constant<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"os"
	"runtime"
	"time"

	keybase1 "github.com/keybase/client/go/protocol"
)

const (
	DevelServerURI      = "http://localhost:3000"
	StagingServerURI    = "https://stage0.keybase.io"
	ProductionServerURI = "https://keybase.io"
	TorServerURI        = "http://fncuwbiisyh6ak3i.onion"
)

type RunMode string

var TorProxy = "localhost:9050"

const (
	DevelRunMode      RunMode = "devel"
	StagingRunMode            = "staging"
	ProductionRunMode         = "prod"
	RunModeError              = "error"
	NoRunMode                 = ""
)

var RunModes = []RunMode{DevelRunMode, StagingRunMode, ProductionRunMode}

var ServerLookup = map[RunMode]string{
	DevelRunMode:      DevelServerURI,
	StagingRunMode:    StagingServerURI,
	ProductionRunMode: ProductionServerURI,
}

const (
	ConfigFile  = "config.json"
	SessionFile = "session.json"
	DBFile      = "keybase.leveldb"
	SocketFile  = "keybased.sock"
	PIDFile     = "keybased.pid"

	SecretKeyringTemplate = "secretkeys.%u.mpack"

	APIVersion       = "1.0"
	APIURIPathPrefix = "/_/api/" + APIVersion
	DaemonPort       = 40933
	GoClientID       = "keybase.io go client"
	IdentifyAs       = GoClientID + " v" + Version + " " + runtime.GOOS
)

var UserAgent = "Keybase/" + Version + " (" + runtime.Version() + " on " + runtime.GOOS + ")"

const (
	PermFile          os.FileMode = 0600
	PermDir           os.FileMode = 0700
	UmaskablePermFile os.FileMode = 0666
)

const (
	UserCacheMaxAge      = 5 * time.Minute
	PGPFingerprintHexLen = 40

	ProofCacheSize      = 0x1000
	ProofCacheLongDur   = 6 * time.Hour
	ProofCacheMediumDur = 30 * time.Minute
	ProofCacheShortDur  = 1 * time.Minute

	SigShortIDBytes = 27
)

var MerkleProdKIDs = []string{
	"010159baae6c7d43c66adf8fb7bb2b8b4cbe408c062cfc369e693ccb18f85631dbcd0a",
}
var MerkleTestKIDs = []string{
	"0101be58b6c82db64f6ccabb05088db443c69f87d5d48857d709ed6f73948dabe67d0a",
}
var MerkleStagingKIDs = []string{
	"0101bed85ce72cc315828367c28b41af585b6b7d95646a62ca829691d70f49184fa70a",
}

const (
	KeybaseKIDV1       = 1 // Uses SHA-256
	KeybaseSignatureV1 = 1
	OneYearInSeconds   = 24 * 60 * 60 * 365

	SigExpireIn       = OneYearInSeconds * 16 // 16 years
	NaclEdDSAExpireIn = OneYearInSeconds * 16 // 16 years
	NaclDHExpireIn    = OneYearInSeconds * 16 // 16 years
	KeyExpireIn       = OneYearInSeconds * 16 // 16 years
	SubkeyExpireIn    = OneYearInSeconds * 16 // 16 years
	AuthExpireIn      = OneYearInSeconds      // 1 year
)

// Status codes.  This list should match keybase/lib/constants.iced.
const (
	SCOk                     = 0
	SCLoginRequired          = 201
	SCBadSession             = 202
	SCBadLoginPassword       = 204
	SCNotFound               = 205
	SCGeneric                = 218
	SCAlreadyLoggedIn        = 235
	SCCanceled               = 237
	SCReloginRequired        = 274
	SCBadSignupUsernameTaken = 701
	SCKeyNotFound            = 901
	SCKeyInUse               = 907
	SCKeyBadGen              = 913
	SCKeyNoSecret            = 914
	SCKeyBadUIDs             = 915
	SCKeyNoActive            = 916
	SCKeyNoSig               = 917
	SCKeyBadSig              = 918
	SCKeyBadEldest           = 919
	SCKeyNoEldest            = 920
	SCKeyDuplicateUpdate     = 921
	SCSibkeyAlreadyExists    = 922
	SCDecryptionKeyNotFound  = 924
	SCBadTrackSession        = 1301
	SCDeviceNotFound         = 1409
	SCDeviceMismatch         = 1410
	SCDeviceRequired         = 1411
	SCStreamExists           = 1501
	SCStreamNotFound         = 1502
	SCStreamWrongKind        = 1503
	SCStreamEOF              = 1504
	SCAPINetworkError        = 1601
	SCTimeout                = 1602
	SCProofError             = 1701
	SCIdentificationExpired  = 1702
	SCSelfNotFound           = 1703
	SCBadKexPhrase           = 1704
	SCNoUIDelegation         = 1705
)

const (
	IDSuffixKID = 0x0a
)

const (
	MerkleTreeNode = 1
	MerkleTreeLeaf = 2
)

type LinkType string
type DelegationType LinkType

const (
	AuthenticationType    LinkType = "auth"
	CryptocurrencyType             = "cryptocurrency"
	RevokeType                     = "revoke"
	TrackType                      = "track"
	UntrackType                    = "untrack"
	UpdatePassphraseType           = "update_passphrase_hash"
	WebServiceBindingType          = "web_service_binding"

	EldestType    DelegationType = "eldest"
	PGPUpdateType                = "pgp_update"
	SibkeyType                   = "sibkey"
	SubkeyType                   = "subkey"
)

const (
	SigTypeNone           = 0
	SigTypeSelfSig        = 1
	SigTypeRemoteProof    = 2
	SigTypeTrack          = 3
	SigTypeUntrack        = 4
	SigTypeRevoke         = 5
	SigTypeCryptocurrency = 6
	SigTypeAnnouncement   = 7
)

type KeyType int

const (
	KeyTypeNone                  KeyType = 0
	KeyTypeOpenPGPPublic                 = 1
	KeyTypeP3skbPrivate                  = 2
	KeyTypeKbNaclEddsa                   = 3
	KeyTypeKbNaclDH                      = 4
	KeyTypeKbNaclEddsaServerHalf         = 5
	KeyTypeKbNaclDHServerHalf            = 6
)

const (
	DeviceStatusNone    = 0
	DeviceStatusActive  = 1
	DeviceStatusDefunct = 2
)

// these strings need to match the keys in
// keybase/lib_public/public_constants.iced ->
// public_constants.device.type
const (
	DeviceTypeDesktop = "desktop"
	DeviceTypeMobile  = "mobile"
	DeviceTypePaper   = "backup"
)

const DownloadURL = "https://keybase.io/download"

var PGPVersion = "Keybase Go " + Version + " (" + runtime.GOOS + ")"

var PGPArmorHeaders = map[string]string{
	"Version": PGPVersion,
	"Comment": DownloadURL,
}

var RemoteServiceTypes = map[string]keybase1.ProofType{
	"keybase":    keybase1.ProofType_KEYBASE,
	"twitter":    keybase1.ProofType_TWITTER,
	"github":     keybase1.ProofType_GITHUB,
	"reddit":     keybase1.ProofType_REDDIT,
	"coinbase":   keybase1.ProofType_COINBASE,
	"hackernews": keybase1.ProofType_HACKERNEWS,
	"https":      keybase1.ProofType_GENERIC_WEB_SITE,
	"http":       keybase1.ProofType_GENERIC_WEB_SITE,
	"dns":        keybase1.ProofType_DNS,
	"rooter":     keybase1.ProofType_ROOTER,
}

var RemoteServiceOrder = []keybase1.ProofType{
	keybase1.ProofType_KEYBASE,
	keybase1.ProofType_TWITTER,
	keybase1.ProofType_GITHUB,
	keybase1.ProofType_REDDIT,
	keybase1.ProofType_COINBASE,
	keybase1.ProofType_HACKERNEWS,
	keybase1.ProofType_GENERIC_WEB_SITE,
	keybase1.ProofType_ROOTER,
}

const CanonicalHost = "keybase.io"

const (
	HTTPDefaultTimeout = 60 * time.Second
	HTTPPollMaximum    = 5 * time.Second
)

// Packet tags for OpenPGP and also Keybase packets
const (
	KeybasePacketV1 = 1
	TagP3skb        = 513
	TagSignature    = 514
	TagEncryption   = 515
)

const (
	KIDPGPBase    AlgoType = 0x00
	KIDPGPRsa              = 0x1
	KIDPGPElgamal          = 0x10
	KIDPGPDsa              = 0x11
	KIDPGPEcdh             = 0x12
	KIDPGPEcdsa            = 0x13
	KIDNaclEddsa           = 0x20
	KIDNaclDH              = 0x21
)

// OpenPGP hash IDs, taken from http://tools.ietf.org/html/rfc4880#section-9.4
const (
	HashPGPMd5       = 1
	HashPGPSha1      = 2
	HashPGPRipemd160 = 3
	HashPGPSha256    = 8
	HashPGPSha384    = 9
	HashPGPSha512    = 10
	HashPGPSha224    = 11
)

const (
	SigKbEddsa = KIDNaclEddsa
)

const (
	ServerUpdateLag = time.Minute
)

// key_revocation_types
const (
	RevSimpleDelete = 0
	RevFull         = 1
	RevDated        = 2
)

type KeyStatus int

const (
	KeyUncancelled KeyStatus = iota
	KeyRevoked
	KeyDeleted
	KeySuperseded
)

type KeyRole int

const (
	DLGNone KeyRole = iota
	DLGSibkey
	DLGSubkey
)

const (
	KexScryptCost       = 32768
	KexScryptR          = 8
	KexScryptP          = 1
	KexScryptKeylen     = 32
	KexSessionIDEntropy = 65 // kex doc specifies 65 bits of entropy
)

const (
	Kex2PhraseEntropy = 88
	Kex2ScryptCost    = 1 << 17
	Kex2ScryptR       = 8
	Kex2ScryptP       = 1
	Kex2ScryptKeylen  = 32
)

const (
	PaperKeyScryptCost    = 32768
	PaperKeyScryptR       = 8
	PaperKeyScryptP       = 1
	PaperKeyScryptKeylen  = 128
	PaperKeySecretEntropy = 117
	PaperKeyIDBits        = 22
	PaperKeyVersionBits   = 4
	PaperKeyVersion       = 0
)

const UserSummaryLimit = 500 // max number of user summaries in one request

const MinPassphraseLength = 12

type KexRole int

const (
	KexRoleProvisioner KexRole = iota
	KexRoleProvisionee
)

const (
	IdentifySourceKBFS = "kbfs"
)
<|endoftext|>


<commit_before>// Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pserver

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"sync"
	"time"

	uuid "github.com/satori/go.uuid"

	log "github.com/inconshreveable/log15"
)

// ElementType is the type of elements of a Parameter.
type ElementType int

// ErrCheckpointNotFound indicates that the pserver checkpoint could
// not be found.
var ErrCheckpointNotFound = errors.New("checkpoint not found")

// RPC error message.
const (
	AlreadyInitialized = "pserver already initialized"
	Uninitialized      = "pserver not fully initialized"
	WrongChecksum      = "checkpoint file checksum validation failed"
)

// Supported element types.
const (
	Int32 ElementType = iota
	UInt32
	Int64
	UInt64
	Float32
	Float64
)

// Parameter is a piece of data to sync with the parameter server.
type Parameter struct {
	Name        string
	ElementType ElementType
	Content     []byte
}

// ParameterWithConfig contains the parameter and the configuration.
type ParameterWithConfig struct {
	Param  Parameter
	Config []byte // parameter configuration in Proto Buffer format
}

// checkpointMeta saves checkpoint metadata
type checkpointMeta struct {
	UUID      string `json:"uuid"`
	Path      string `json:"path"`
	MD5       string `json:"md5"`
	Timestamp int64  `json:"timestamp"`
}

// Checkpoint is the pserver shard persist in file.
type Checkpoint []parameterCheckpoint

// Gradient is the gradient of the parameter.
type Gradient Parameter

// Service is the RPC service for pserver.
type Service struct {
	initialized        chan struct{}
	idx                int
	checkpointInterval time.Duration
	checkpointPath     string
	client             *EtcdClient

	mu     sync.Mutex
	optMap map[string]*optimizer
}

// parameterCheckpoint saves parameter checkpoint.
type parameterCheckpoint struct {
	ParameterWithConfig
	State []byte
}

func loadMeta(e *EtcdClient, idx int) (meta checkpointMeta, err error) {
	v, err := e.GetKey(PsCheckpoint+strconv.Itoa(idx), 3*time.Second)
	if err != nil {
		return
	}

	if len(v) == 0 {
		err = ErrCheckpointNotFound
		return
	}

	if err = json.Unmarshal(v, &meta); err != nil {
		return
	}

	return
}

// LoadCheckpoint loads checkpoint from file.
func LoadCheckpoint(e *EtcdClient, idx int) (Checkpoint, error) {
	cpMeta, err := loadMeta(e, idx)
	if err != nil {
		return nil, err
	}

	content, err := ioutil.ReadFile(cpMeta.Path)
	if err != nil {
		return nil, err
	}

	// TODO(helin): change MD5 to CRC since CRC is better for file
	// checksum in our use case (emphasize speed over security).
	h := md5.New()
	md5 := hex.EncodeToString(h.Sum(content))
	if md5 != cpMeta.MD5 {
		return nil, errors.New(WrongChecksum)
	}

	dec := gob.NewDecoder(bytes.NewReader(content))
	var cp Checkpoint
	if err = dec.Decode(&cp); err != nil {
		return nil, err
	}
	return cp, nil
}

// NewService creates a new service, will bypass etcd registration if no
// endpoints specified. It will recovery from checkpoint file if a exists a specified checkpoint.
func NewService(idx int, interval time.Duration, path string, client *EtcdClient, cp Checkpoint) (*Service, error) {
	s := &Service{
		idx:                idx,
		checkpointInterval: interval,
		checkpointPath:     path,
		client:             client,
	}
	s.optMap = make(map[string]*optimizer)
	s.initialized = make(chan struct{})

	if cp != nil {
		for _, item := range cp {
			p := ParameterWithConfig{
				Param:  item.Param,
				Config: item.Config,
			}
			s.optMap[p.Param.Name] = newOptimizer(p, item.State)
		}
	}
	return s, nil
}

// InitParam initializes a parameter.
func (s *Service) InitParam(paramWithConfigs ParameterWithConfig, _ *int) error {
	select {
	case <-s.initialized:
		return errors.New(AlreadyInitialized)
	default:
	}

	// TODO(helin): parse parameter config

	s.mu.Lock()
	defer s.mu.Unlock()

	// TODO(helin): check if paramWithConfigs.Param.Content is
	// properly memory aligned, if not, make copy to a memory
	// aligned region.
	s.optMap[paramWithConfigs.Param.Name] = newOptimizer(paramWithConfigs, nil)
	return nil
}

// FinishInitParams tells the parameter server that the parameter
// initialization has finished.
func (s *Service) FinishInitParams(_ int, _ *int) error {
	select {
	case <-s.initialized:
		return errors.New(AlreadyInitialized)
	default:
	}

	close(s.initialized)
	go func() {
		t := time.Tick(s.checkpointInterval)
		for range t {
			err := s.checkpoint()
			if err != nil {
				log.Error("finish init params error", log.Ctx{"error": err})
			}
		}
	}()
	return nil
}

// SendGrad sends gradient to parameter servers for parameter
// optimization.
func (s *Service) SendGrad(g Gradient, _ *int) error {
	select {
	case <-s.initialized:
	default:
		return errors.New(Uninitialized)
	}

	s.mu.Lock()
	defer s.mu.Unlock()

	o, ok := s.optMap[g.Name]
	if !ok {
		return fmt.Errorf("parameter: %s does not exist", g.Name)
	}

	return o.UpdateParameter(g)
}

// GetParam gets parameters from the parameter server.
func (s *Service) GetParam(name string, parameter *Parameter) error {
	<-s.initialized
	s.mu.Lock()
	defer s.mu.Unlock()

	opt, ok := s.optMap[name]
	if !ok {
		return fmt.Errorf("parameter: %s does not exist", name)
	}

	// The parameter content (a byte slice) may change
	// during RPC serialization due to write from other
	// goroutine, we allow it since mini-batch based deep
	// learning optimization methods are stochastic in
	// nature. This race condition is allowed deliberately
	// to save the program from making a copy of the
	// parameter content.
	parameter.Name = name
	parameter.ElementType = opt.elementType
	parameter.Content = opt.GetWeights()
	return nil
}

func traceTime(start time.Time, name string) {
	elapsed := time.Since(start)
	log.Info("time elapsed", log.Ctx{"name": name, "elapsed": elapsed})
}

// checkpoint saves checkpoint to disk.
//
// checkpoint should be only called after the parameters are
// initialized.
func (s *Service) checkpoint() (err error) {
	log.Info("Begin save checkpoint.")
	defer traceTime(time.Now(), "save checkpoint")

	s.mu.Lock()
	cp := make([]parameterCheckpoint, len(s.optMap))
	index := 0
	// TODO(helin): write checkpoint incrementally to reduce memory
	// footprint during checkpoint.
	for name, opt := range s.optMap {
		var pc parameterCheckpoint
		pc.Param.Name = name
		pc.Param.ElementType = opt.elementType
		pc.Param.Content = opt.GetWeights()
		pc.Config = opt.config
		pc.State = opt.GetStates()
		cp[index] = pc
		index++
	}
	s.mu.Unlock()

	var buf bytes.Buffer
	encoder := gob.NewEncoder(&buf)
	err = encoder.Encode(cp)
	if err != nil {
		return
	}

	if _, err = os.Stat(s.checkpointPath); os.IsNotExist(err) {
		err = os.MkdirAll(s.checkpointPath, os.ModePerm)
		if err != nil {
			return
		}
	}

	id := uuid.NewV4().String()
	p := path.Join(s.checkpointPath, id)
	f, err := os.Create(p)
	if err != nil {
		return
	}

	defer func() {
		closeErr := f.Close()
		if closeErr != nil {
			if err != nil {
				log.Error("error close checkpoint file", log.Ctx{"error": closeErr})
			} else {
				// Set closeErr as return value.
				err = closeErr
			}
		}
	}()

	writer := bufio.NewWriter(f)
	_, err = writer.Write(buf.Bytes())
	if err != nil {
		return
	}

	err = writer.Flush()
	if err != nil {
		return
	}

	oldMeta, err := loadMeta(s.client, s.idx)
	if err == ErrCheckpointNotFound {
		log.Info("Do not have existing checkpoint.")
		err = nil
	}

	if err != nil {
		return
	}

	h := md5.New()
	md5 := hex.EncodeToString(h.Sum(buf.Bytes()))
	cpMeta := checkpointMeta{
		UUID:      id,
		Timestamp: time.Now().UnixNano(),
		MD5:       md5,
		Path:      p,
	}

	json, err := json.Marshal(cpMeta)
	if err != nil {
		return
	}

	err = s.client.PutKey(PsCheckpoint+strconv.Itoa(s.idx), json, 3*time.Second, false)
	if err != nil {
		return
	}

	if oldMeta.Path != "" {
		rmErr := os.Remove(oldMeta.Path)
		if rmErr != nil {
			// log error, but still treat checkpoint as
			// successful.
			log.Error("remove old meta file error", log.Ctx{"error": rmErr})
		}
	}

	return
}
<commit_msg>add detailed log for the pserver<commit_after>// Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

// http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pserver

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"encoding/gob"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"sync"
	"time"

	uuid "github.com/satori/go.uuid"

	log "github.com/inconshreveable/log15"
)

// ElementType is the type of elements of a Parameter.
type ElementType int

// ErrCheckpointNotFound indicates that the pserver checkpoint could
// not be found.
var ErrCheckpointNotFound = errors.New("checkpoint not found")

// RPC error message.
const (
	AlreadyInitialized = "pserver already initialized"
	Uninitialized      = "pserver not fully initialized"
	WrongChecksum      = "checkpoint file checksum validation failed"
)

// Supported element types.
const (
	Int32 ElementType = iota
	UInt32
	Int64
	UInt64
	Float32
	Float64
)

// Parameter is a piece of data to sync with the parameter server.
type Parameter struct {
	Name        string
	ElementType ElementType
	Content     []byte
}

// ParameterWithConfig contains the parameter and the configuration.
type ParameterWithConfig struct {
	Param  Parameter
	Config []byte // parameter configuration in Proto Buffer format
}

// checkpointMeta saves checkpoint metadata
type checkpointMeta struct {
	UUID      string `json:"uuid"`
	Path      string `json:"path"`
	MD5       string `json:"md5"`
	Timestamp int64  `json:"timestamp"`
}

// Checkpoint is the pserver shard persist in file.
type Checkpoint []parameterCheckpoint

// Gradient is the gradient of the parameter.
type Gradient Parameter

// Service is the RPC service for pserver.
type Service struct {
	initialized        chan struct{}
	idx                int
	checkpointInterval time.Duration
	checkpointPath     string
	client             *EtcdClient

	mu     sync.Mutex
	optMap map[string]*optimizer
}

// parameterCheckpoint saves parameter checkpoint.
type parameterCheckpoint struct {
	ParameterWithConfig
	State []byte
}

func loadMeta(e *EtcdClient, idx int) (meta checkpointMeta, err error) {
	v, err := e.GetKey(PsCheckpoint+strconv.Itoa(idx), 3*time.Second)
	if err != nil {
		return
	}

	if len(v) == 0 {
		err = ErrCheckpointNotFound
		return
	}

	if err = json.Unmarshal(v, &meta); err != nil {
		return
	}

	return
}

// LoadCheckpoint loads checkpoint from file.
func LoadCheckpoint(e *EtcdClient, idx int) (Checkpoint, error) {
	log.Info("Loading checkpoint", "pserver index", idx)
	defer traceTime(time.Now(), "load checkpoint")

	cpMeta, err := loadMeta(e, idx)
	if err != nil {
		return nil, err
	}

	content, err := ioutil.ReadFile(cpMeta.Path)
	if err != nil {
		return nil, err
	}

	// TODO(helin): change MD5 to CRC since CRC is better for file
	// checksum in our use case (emphasize speed over security).
	h := md5.New()
	md5 := hex.EncodeToString(h.Sum(content))
	if md5 != cpMeta.MD5 {
		return nil, errors.New(WrongChecksum)
	}

	dec := gob.NewDecoder(bytes.NewReader(content))
	var cp Checkpoint
	if err = dec.Decode(&cp); err != nil {
		return nil, err
	}
	return cp, nil
}

// NewService creates a new service, will bypass etcd registration if no
// endpoints specified. It will recovery from checkpoint file if a exists a specified checkpoint.
func NewService(idx int, interval time.Duration, path string, client *EtcdClient, cp Checkpoint) (*Service, error) {
	s := &Service{
		idx:                idx,
		checkpointInterval: interval,
		checkpointPath:     path,
		client:             client,
	}
	s.optMap = make(map[string]*optimizer)
	s.initialized = make(chan struct{})

	if cp != nil {
		for _, item := range cp {
			p := ParameterWithConfig{
				Param:  item.Param,
				Config: item.Config,
			}
			s.optMap[p.Param.Name] = newOptimizer(p, item.State)
		}
	}
	return s, nil
}

// InitParam initializes a parameter.
func (s *Service) InitParam(paramWithConfigs ParameterWithConfig, _ *int) error {
	select {
	case <-s.initialized:
		log.Warn("init param called but parameters already initialized.")
		return errors.New(AlreadyInitialized)
	default:
	}

	// TODO(helin): parse parameter config

	s.mu.Lock()
	defer s.mu.Unlock()

	// TODO(helin): check if paramWithConfigs.Param.Content is
	// properly memory aligned, if not, make copy to a memory
	// aligned region.
	s.optMap[paramWithConfigs.Param.Name] = newOptimizer(paramWithConfigs, nil)
	log.Info(
		"init parameter",
		"name", paramWithConfigs.Param.Name,
		"config len", len(paramWithConfigs.Config),
		"param len", len(paramWithConfigs.Param.Content),
		"type", paramWithConfigs.Param.ElementType,
	)
	return nil
}

// FinishInitParams tells the parameter server that the parameter
// initialization has finished.
func (s *Service) FinishInitParams(_ int, _ *int) error {
	select {
	case <-s.initialized:
		log.Warn("finished init param called but parameters already initialized.")
		return errors.New(AlreadyInitialized)
	default:
	}

	close(s.initialized)
	go func() {
		t := time.Tick(s.checkpointInterval)
		for range t {
			err := s.checkpoint()
			if err != nil {
				log.Error("finish init params error", log.Ctx{"error": err})
			}
		}
	}()

	log.Info("init parameter finished.")
	return nil
}

// SendGrad sends gradient to parameter servers for parameter
// optimization.
func (s *Service) SendGrad(g Gradient, _ *int) error {
	select {
	case <-s.initialized:
	default:
		log.Warn("received gradient before initialization.", "name", g.Name, "size", len(g.Content), "type", g.ElementType)
		return errors.New(Uninitialized)
	}

	s.mu.Lock()
	defer s.mu.Unlock()

	o, ok := s.optMap[g.Name]
	if !ok {
		return fmt.Errorf("parameter: %s does not exist", g.Name)
	}

	log.Info("received gradient from trainer, updating gradient.", "name", g.Name, "size", len(g.Content), "type", g.ElementType)
	return o.UpdateParameter(g)
}

// GetParam gets parameters from the parameter server.
func (s *Service) GetParam(name string, parameter *Parameter) error {
	<-s.initialized
	s.mu.Lock()
	defer s.mu.Unlock()

	opt, ok := s.optMap[name]
	if !ok {
		log.Warn("trainer wants to get a parameter that does not exist.", "name", name)
		return fmt.Errorf("parameter: %s does not exist", name)
	}

	// The parameter content (a byte slice) may change
	// during RPC serialization due to write from other
	// goroutine, we allow it since mini-batch based deep
	// learning optimization methods are stochastic in
	// nature. This race condition is allowed deliberately
	// to save the program from making a copy of the
	// parameter content.
	parameter.Name = name
	parameter.ElementType = opt.elementType
	parameter.Content = opt.GetWeights()
	log.Info("sending parameter to the trainer", "name", parameter.Name, "size", len(parameter.Content), "type", parameter.ElementType)
	return nil
}

func traceTime(start time.Time, name string) {
	elapsed := time.Since(start)
	log.Info("time elapsed", log.Ctx{"name": name, "elapsed": elapsed})
}

// checkpoint saves checkpoint to disk.
//
// checkpoint should be only called after the parameters are
// initialized.
func (s *Service) checkpoint() (err error) {
	log.Info("Begin save checkpoint.")
	defer traceTime(time.Now(), "save checkpoint")

	s.mu.Lock()
	cp := make([]parameterCheckpoint, len(s.optMap))
	index := 0
	// TODO(helin): write checkpoint incrementally to reduce memory
	// footprint during checkpoint.
	for name, opt := range s.optMap {
		var pc parameterCheckpoint
		pc.Param.Name = name
		pc.Param.ElementType = opt.elementType
		pc.Param.Content = opt.GetWeights()
		pc.Config = opt.config
		pc.State = opt.GetStates()
		cp[index] = pc
		index++
	}
	s.mu.Unlock()

	var buf bytes.Buffer
	encoder := gob.NewEncoder(&buf)
	err = encoder.Encode(cp)
	if err != nil {
		return
	}

	if _, err = os.Stat(s.checkpointPath); os.IsNotExist(err) {
		err = os.MkdirAll(s.checkpointPath, os.ModePerm)
		if err != nil {
			return
		}
	}

	id := uuid.NewV4().String()
	p := path.Join(s.checkpointPath, id)
	f, err := os.Create(p)
	if err != nil {
		return
	}

	defer func() {
		closeErr := f.Close()
		if closeErr != nil {
			if err != nil {
				log.Error("error close checkpoint file", log.Ctx{"error": closeErr})
			} else {
				// Set closeErr as return value.
				err = closeErr
			}
		}
	}()

	writer := bufio.NewWriter(f)
	_, err = writer.Write(buf.Bytes())
	if err != nil {
		return
	}

	err = writer.Flush()
	if err != nil {
		return
	}

	oldMeta, err := loadMeta(s.client, s.idx)
	if err == ErrCheckpointNotFound {
		log.Info("Do not have existing checkpoint.")
		err = nil
	}

	if err != nil {
		return
	}

	h := md5.New()
	md5 := hex.EncodeToString(h.Sum(buf.Bytes()))
	cpMeta := checkpointMeta{
		UUID:      id,
		Timestamp: time.Now().UnixNano(),
		MD5:       md5,
		Path:      p,
	}

	json, err := json.Marshal(cpMeta)
	if err != nil {
		return
	}

	err = s.client.PutKey(PsCheckpoint+strconv.Itoa(s.idx), json, 3*time.Second, false)
	if err != nil {
		return
	}

	if oldMeta.Path != "" {
		rmErr := os.Remove(oldMeta.Path)
		if rmErr != nil {
			// log error, but still treat checkpoint as
			// successful.
			log.Error("remove old meta file error", log.Ctx{"error": rmErr})
		}
	}

	return
}
<|endoftext|>


<commit_before>package cmds

import (
	"fmt"

	"github.com/urfave/cli"
)

type Server struct {
	ClusterCIDR              string
	AgentToken               string
	AgentTokenFile           string
	Token                    string
	TokenFile                string
	ServiceCIDR              string
	ClusterDNS               string
	ClusterDomain            string
	HTTPSPort                int
	DataDir                  string
	DisableAgent             bool
	KubeConfigOutput         string
	KubeConfigMode           string
	TLSSan                   cli.StringSlice
	BindAddress              string
	ExtraAPIArgs             cli.StringSlice
	ExtraSchedulerArgs       cli.StringSlice
	ExtraControllerArgs      cli.StringSlice
	ExtraCloudControllerArgs cli.StringSlice
	Rootless                 bool
	StorageEndpoint          string
	StorageCAFile            string
	StorageCertFile          string
	StorageKeyFile           string
	AdvertiseIP              string
	AdvertisePort            int
	DisableScheduler         bool
	ServerURL                string
	FlannelBackend           string
	DefaultLocalStoragePath  string
	DisableCCM               bool
	DisableNPC               bool
	ClusterInit              bool
	ClusterReset             bool
}

var ServerConfig Server

func NewServerCommand(action func(*cli.Context) error) cli.Command {
	return cli.Command{
		Name:      "server",
		Usage:     "Run management server",
		UsageText: appName + " server [OPTIONS]",
		Action:    action,
		Flags: []cli.Flag{
			VLevel,
			VModule,
			LogFile,
			AlsoLogToStderr,
			cli.StringFlag{
				Name:        "bind-address",
				Usage:       "(listener) k3s bind address (default: 0.0.0.0)",
				Destination: &ServerConfig.BindAddress,
			},
			cli.IntFlag{
				Name:        "https-listen-port",
				Usage:       "(listener) HTTPS listen port",
				Value:       6443,
				Destination: &ServerConfig.HTTPSPort,
			},
			cli.StringFlag{
				Name:        "advertise-address",
				Usage:       "(listener) IP address that apiserver uses to advertise to members of the cluster (default: node-external-ip/node-ip)",
				Destination: &ServerConfig.AdvertiseIP,
			},
			cli.IntFlag{
				Name:        "advertise-port",
				Usage:       "(listener) Port that apiserver uses to advertise to members of the cluster (default: listen-port)",
				Destination: &ServerConfig.AdvertisePort,
			},
			cli.StringSliceFlag{
				Name:  "tls-san",
				Usage: "(listener) Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
				Value: &ServerConfig.TLSSan,
			},
			cli.StringFlag{
				Name:        "data-dir,d",
				Usage:       "(data) Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root",
				Destination: &ServerConfig.DataDir,
			},
			cli.StringFlag{
				Name:        "cluster-cidr",
				Usage:       "(networking) Network CIDR to use for pod IPs",
				Destination: &ServerConfig.ClusterCIDR,
				Value:       "10.42.0.0/16",
			},
			cli.StringFlag{
				Name:        "service-cidr",
				Usage:       "(networking) Network CIDR to use for services IPs",
				Destination: &ServerConfig.ServiceCIDR,
				Value:       "10.43.0.0/16",
			},
			cli.StringFlag{
				Name:        "cluster-dns",
				Usage:       "(networking) Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)",
				Destination: &ServerConfig.ClusterDNS,
				Value:       "",
			},
			cli.StringFlag{
				Name:        "cluster-domain",
				Usage:       "(networking) Cluster Domain",
				Destination: &ServerConfig.ClusterDomain,
				Value:       "cluster.local",
			},
			cli.StringFlag{
				Name:        "flannel-backend",
				Usage:       fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', or 'flannel'"),
				Destination: &ServerConfig.FlannelBackend,
				Value:       "vxlan",
			},
			cli.StringFlag{
				Name:        "token,t",
				Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
				Destination: &ServerConfig.Token,
				EnvVar:      "K3S_CLUSTER_SECRET,K3S_TOKEN",
			},
			cli.StringFlag{
				Name:        "token-file",
				Usage:       "(cluster) File containing the cluster-secret/token",
				Destination: &ServerConfig.TokenFile,
				EnvVar:      "K3S_TOKEN_FILE",
			},
			cli.StringFlag{
				Name:        "agent-token",
				Usage:       "(cluster) Shared secret used to join agents to the cluster, but not agents",
				Destination: &ServerConfig.AgentToken,
				EnvVar:      "K3S_AGENT_TOKEN",
			},
			cli.StringFlag{
				Name:        "agent-token-file",
				Usage:       "(cluster) File containing the agent secret",
				Destination: &ServerConfig.AgentTokenFile,
				EnvVar:      "K3S_AGENT_TOKEN_FILE",
			},
			cli.StringFlag{
				Name:        "server,s",
				Usage:       "(cluster) Server to connect to, used to join a cluster",
				EnvVar:      "K3S_URL",
				Destination: &ServerConfig.ServerURL,
			},
			cli.BoolFlag{
				Name:        "cluster-init",
				Hidden:      hideDqlite,
				Usage:       "(cluster) Initialize new cluster master",
				EnvVar:      "K3S_CLUSTER_INIT",
				Destination: &ServerConfig.ClusterInit,
			},
			cli.BoolFlag{
				Name:        "cluster-reset",
				Hidden:      hideDqlite,
				Usage:       "(cluster) Forget all peers and become a single cluster new cluster master",
				EnvVar:      "K3S_CLUSTER_RESET",
				Destination: &ServerConfig.ClusterReset,
			},
			cli.StringFlag{
				Name:        "write-kubeconfig,o",
				Usage:       "(client) Write kubeconfig for admin client to this file",
				Destination: &ServerConfig.KubeConfigOutput,
				EnvVar:      "K3S_KUBECONFIG_OUTPUT",
			},
			cli.StringFlag{
				Name:        "write-kubeconfig-mode",
				Usage:       "(client) Write kubeconfig with this mode",
				Destination: &ServerConfig.KubeConfigMode,
				EnvVar:      "K3S_KUBECONFIG_MODE",
			},
			cli.StringSliceFlag{
				Name:  "kube-apiserver-arg",
				Usage: "(flags) Customized flag for kube-apiserver process",
				Value: &ServerConfig.ExtraAPIArgs,
			},
			cli.StringSliceFlag{
				Name:  "kube-scheduler-arg",
				Usage: "(flags) Customized flag for kube-scheduler process",
				Value: &ServerConfig.ExtraSchedulerArgs,
			},
			cli.StringSliceFlag{
				Name:  "kube-controller-manager-arg",
				Usage: "(flags) Customized flag for kube-controller-manager process",
				Value: &ServerConfig.ExtraControllerArgs,
			},
			cli.StringSliceFlag{
				Name:  "kube-cloud-controller-manager-arg",
				Usage: "(flags) Customized flag for kube-cloud-controller-manager process",
				Value: &ServerConfig.ExtraCloudControllerArgs,
			},
			cli.StringFlag{
				Name:        "storage-endpoint",
				Usage:       "(db) Specify etcd, Mysql, Postgres, or Sqlite (default) data source name",
				Destination: &ServerConfig.StorageEndpoint,
				EnvVar:      "K3S_STORAGE_ENDPOINT",
			},
			cli.StringFlag{
				Name:        "storage-cafile",
				Usage:       "(db) SSL Certificate Authority file used to secure storage backend communication",
				Destination: &ServerConfig.StorageCAFile,
				EnvVar:      "K3S_STORAGE_CAFILE",
			},
			cli.StringFlag{
				Name:        "storage-certfile",
				Usage:       "(db) SSL certification file used to secure storage backend communication",
				Destination: &ServerConfig.StorageCertFile,
				EnvVar:      "K3S_STORAGE_CERTFILE",
			},
			cli.StringFlag{
				Name:        "storage-keyfile",
				Usage:       "(db) SSL key file used to secure storage backend communication",
				Destination: &ServerConfig.StorageKeyFile,
				EnvVar:      "K3S_STORAGE_KEYFILE",
			},
			cli.StringFlag{
				Name:        "default-local-storage-path",
				Usage:       "(storage) Default local storage path for local provisioner storage class",
				Destination: &ServerConfig.DefaultLocalStoragePath,
			},
			cli.StringSliceFlag{
				Name:  "no-deploy",
				Usage: "(components) Do not deploy packaged components (valid items: coredns, servicelb, traefik, local-storage, metrics-server)",
			},
			cli.BoolFlag{
				Name:        "disable-scheduler",
				Usage:       "(components) Disable Kubernetes default scheduler",
				Destination: &ServerConfig.DisableScheduler,
			},
			cli.BoolFlag{
				Name:        "disable-cloud-controller",
				Usage:       "(components) Disable k3s default cloud controller manager",
				Destination: &ServerConfig.DisableCCM,
			},
			cli.BoolFlag{
				Name:        "disable-network-policy",
				Usage:       "(components) Disable k3s default network policy controller",
				Destination: &ServerConfig.DisableNPC,
			},
			NodeNameFlag,
			NodeLabels,
			NodeTaints,
			DockerFlag,
			CRIEndpointFlag,
			PauseImageFlag,
			PrivateRegistryFlag,
			NodeIPFlag,
			NodeExternalIPFlag,
			ResolvConfFlag,
			FlannelIfaceFlag,
			FlannelConfFlag,
			ExtraKubeletArgs,
			ExtraKubeProxyArgs,
			cli.BoolFlag{
				Name:        "rootless",
				Usage:       "(experimental) Run rootless",
				Destination: &ServerConfig.Rootless,
			},

			// Hidden/Deprecated flags below

			FlannelFlag,
			cli.StringFlag{
				Name:        "cluster-secret",
				Usage:       "(deprecated) use --token",
				Destination: &ServerConfig.Token,
				EnvVar:      "K3S_CLUSTER_SECRET",
			},
			cli.BoolFlag{
				Name:        "disable-agent",
				Usage:       "Do not run a local agent and register a local kubelet",
				Hidden:      true,
				Destination: &ServerConfig.DisableAgent,
			},
			cli.StringSliceFlag{
				Hidden: true,
				Name:   "kube-controller-arg",
				Usage:  "(flags) Customized flag for kube-controller-manager process",
				Value:  &ServerConfig.ExtraControllerArgs,
			},
			cli.StringSliceFlag{
				Hidden: true,
				Name:   "kube-cloud-controller-arg",
				Usage:  "(flags) Customized flag for kube-cloud-controller-manager process",
				Value:  &ServerConfig.ExtraCloudControllerArgs,
			},
		},
	}
}
<commit_msg>Move server arguments to experimental for dqlite related<commit_after>package cmds

import (
	"fmt"

	"github.com/urfave/cli"
)

type Server struct {
	ClusterCIDR              string
	AgentToken               string
	AgentTokenFile           string
	Token                    string
	TokenFile                string
	ServiceCIDR              string
	ClusterDNS               string
	ClusterDomain            string
	HTTPSPort                int
	DataDir                  string
	DisableAgent             bool
	KubeConfigOutput         string
	KubeConfigMode           string
	TLSSan                   cli.StringSlice
	BindAddress              string
	ExtraAPIArgs             cli.StringSlice
	ExtraSchedulerArgs       cli.StringSlice
	ExtraControllerArgs      cli.StringSlice
	ExtraCloudControllerArgs cli.StringSlice
	Rootless                 bool
	StorageEndpoint          string
	StorageCAFile            string
	StorageCertFile          string
	StorageKeyFile           string
	AdvertiseIP              string
	AdvertisePort            int
	DisableScheduler         bool
	ServerURL                string
	FlannelBackend           string
	DefaultLocalStoragePath  string
	DisableCCM               bool
	DisableNPC               bool
	ClusterInit              bool
	ClusterReset             bool
}

var ServerConfig Server

func NewServerCommand(action func(*cli.Context) error) cli.Command {
	return cli.Command{
		Name:      "server",
		Usage:     "Run management server",
		UsageText: appName + " server [OPTIONS]",
		Action:    action,
		Flags: []cli.Flag{
			VLevel,
			VModule,
			LogFile,
			AlsoLogToStderr,
			cli.StringFlag{
				Name:        "bind-address",
				Usage:       "(listener) k3s bind address (default: 0.0.0.0)",
				Destination: &ServerConfig.BindAddress,
			},
			cli.IntFlag{
				Name:        "https-listen-port",
				Usage:       "(listener) HTTPS listen port",
				Value:       6443,
				Destination: &ServerConfig.HTTPSPort,
			},
			cli.StringFlag{
				Name:        "advertise-address",
				Usage:       "(listener) IP address that apiserver uses to advertise to members of the cluster (default: node-external-ip/node-ip)",
				Destination: &ServerConfig.AdvertiseIP,
			},
			cli.IntFlag{
				Name:        "advertise-port",
				Usage:       "(listener) Port that apiserver uses to advertise to members of the cluster (default: listen-port)",
				Destination: &ServerConfig.AdvertisePort,
			},
			cli.StringSliceFlag{
				Name:  "tls-san",
				Usage: "(listener) Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
				Value: &ServerConfig.TLSSan,
			},
			cli.StringFlag{
				Name:        "data-dir,d",
				Usage:       "(data) Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root",
				Destination: &ServerConfig.DataDir,
			},
			cli.StringFlag{
				Name:        "cluster-cidr",
				Usage:       "(networking) Network CIDR to use for pod IPs",
				Destination: &ServerConfig.ClusterCIDR,
				Value:       "10.42.0.0/16",
			},
			cli.StringFlag{
				Name:        "service-cidr",
				Usage:       "(networking) Network CIDR to use for services IPs",
				Destination: &ServerConfig.ServiceCIDR,
				Value:       "10.43.0.0/16",
			},
			cli.StringFlag{
				Name:        "cluster-dns",
				Usage:       "(networking) Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10)",
				Destination: &ServerConfig.ClusterDNS,
				Value:       "",
			},
			cli.StringFlag{
				Name:        "cluster-domain",
				Usage:       "(networking) Cluster Domain",
				Destination: &ServerConfig.ClusterDomain,
				Value:       "cluster.local",
			},
			cli.StringFlag{
				Name:        "flannel-backend",
				Usage:       fmt.Sprintf("(networking) One of 'none', 'vxlan', 'ipsec', or 'flannel'"),
				Destination: &ServerConfig.FlannelBackend,
				Value:       "vxlan",
			},
			cli.StringFlag{
				Name:        "token,t",
				Usage:       "(cluster) Shared secret used to join a server or agent to a cluster",
				Destination: &ServerConfig.Token,
				EnvVar:      "K3S_CLUSTER_SECRET,K3S_TOKEN",
			},
			cli.StringFlag{
				Name:        "token-file",
				Usage:       "(cluster) File containing the cluster-secret/token",
				Destination: &ServerConfig.TokenFile,
				EnvVar:      "K3S_TOKEN_FILE",
			},
			cli.StringFlag{
				Name:        "write-kubeconfig,o",
				Usage:       "(client) Write kubeconfig for admin client to this file",
				Destination: &ServerConfig.KubeConfigOutput,
				EnvVar:      "K3S_KUBECONFIG_OUTPUT",
			},
			cli.StringFlag{
				Name:        "write-kubeconfig-mode",
				Usage:       "(client) Write kubeconfig with this mode",
				Destination: &ServerConfig.KubeConfigMode,
				EnvVar:      "K3S_KUBECONFIG_MODE",
			},
			cli.StringSliceFlag{
				Name:  "kube-apiserver-arg",
				Usage: "(flags) Customized flag for kube-apiserver process",
				Value: &ServerConfig.ExtraAPIArgs,
			},
			cli.StringSliceFlag{
				Name:  "kube-scheduler-arg",
				Usage: "(flags) Customized flag for kube-scheduler process",
				Value: &ServerConfig.ExtraSchedulerArgs,
			},
			cli.StringSliceFlag{
				Name:  "kube-controller-manager-arg",
				Usage: "(flags) Customized flag for kube-controller-manager process",
				Value: &ServerConfig.ExtraControllerArgs,
			},
			cli.StringSliceFlag{
				Name:  "kube-cloud-controller-manager-arg",
				Usage: "(flags) Customized flag for kube-cloud-controller-manager process",
				Value: &ServerConfig.ExtraCloudControllerArgs,
			},
			cli.StringFlag{
				Name:        "storage-endpoint",
				Usage:       "(db) Specify etcd, Mysql, Postgres, or Sqlite (default) data source name",
				Destination: &ServerConfig.StorageEndpoint,
				EnvVar:      "K3S_STORAGE_ENDPOINT",
			},
			cli.StringFlag{
				Name:        "storage-cafile",
				Usage:       "(db) SSL Certificate Authority file used to secure storage backend communication",
				Destination: &ServerConfig.StorageCAFile,
				EnvVar:      "K3S_STORAGE_CAFILE",
			},
			cli.StringFlag{
				Name:        "storage-certfile",
				Usage:       "(db) SSL certification file used to secure storage backend communication",
				Destination: &ServerConfig.StorageCertFile,
				EnvVar:      "K3S_STORAGE_CERTFILE",
			},
			cli.StringFlag{
				Name:        "storage-keyfile",
				Usage:       "(db) SSL key file used to secure storage backend communication",
				Destination: &ServerConfig.StorageKeyFile,
				EnvVar:      "K3S_STORAGE_KEYFILE",
			},
			cli.StringFlag{
				Name:        "default-local-storage-path",
				Usage:       "(storage) Default local storage path for local provisioner storage class",
				Destination: &ServerConfig.DefaultLocalStoragePath,
			},
			cli.StringSliceFlag{
				Name:  "no-deploy",
				Usage: "(components) Do not deploy packaged components (valid items: coredns, servicelb, traefik, local-storage, metrics-server)",
			},
			cli.BoolFlag{
				Name:        "disable-scheduler",
				Usage:       "(components) Disable Kubernetes default scheduler",
				Destination: &ServerConfig.DisableScheduler,
			},
			cli.BoolFlag{
				Name:        "disable-cloud-controller",
				Usage:       "(components) Disable k3s default cloud controller manager",
				Destination: &ServerConfig.DisableCCM,
			},
			cli.BoolFlag{
				Name:        "disable-network-policy",
				Usage:       "(components) Disable k3s default network policy controller",
				Destination: &ServerConfig.DisableNPC,
			},
			NodeNameFlag,
			NodeLabels,
			NodeTaints,
			DockerFlag,
			CRIEndpointFlag,
			PauseImageFlag,
			PrivateRegistryFlag,
			NodeIPFlag,
			NodeExternalIPFlag,
			ResolvConfFlag,
			FlannelIfaceFlag,
			FlannelConfFlag,
			ExtraKubeletArgs,
			ExtraKubeProxyArgs,
			cli.BoolFlag{
				Name:        "rootless",
				Usage:       "(experimental) Run rootless",
				Destination: &ServerConfig.Rootless,
			},
			cli.StringFlag{
				Name:        "agent-token",
				Usage:       "(experimental/cluster) Shared secret used to join agents to the cluster, but not agents",
				Destination: &ServerConfig.AgentToken,
				EnvVar:      "K3S_AGENT_TOKEN",
			},
			cli.StringFlag{
				Name:        "agent-token-file",
				Usage:       "(experimental/cluster) File containing the agent secret",
				Destination: &ServerConfig.AgentTokenFile,
				EnvVar:      "K3S_AGENT_TOKEN_FILE",
			},
			cli.StringFlag{
				Name:        "server,s",
				Usage:       "(experimental/cluster) Server to connect to, used to join a cluster",
				EnvVar:      "K3S_URL",
				Destination: &ServerConfig.ServerURL,
			},
			cli.BoolFlag{
				Name:        "cluster-init",
				Hidden:      hideDqlite,
				Usage:       "(experimental/cluster) Initialize new cluster master",
				EnvVar:      "K3S_CLUSTER_INIT",
				Destination: &ServerConfig.ClusterInit,
			},
			cli.BoolFlag{
				Name:        "cluster-reset",
				Hidden:      hideDqlite,
				Usage:       "(experimental/cluster) Forget all peers and become a single cluster new cluster master",
				EnvVar:      "K3S_CLUSTER_RESET",
				Destination: &ServerConfig.ClusterReset,
			},

			// Hidden/Deprecated flags below

			FlannelFlag,
			cli.StringFlag{
				Name:        "cluster-secret",
				Usage:       "(deprecated) use --token",
				Destination: &ServerConfig.Token,
				EnvVar:      "K3S_CLUSTER_SECRET",
			},
			cli.BoolFlag{
				Name:        "disable-agent",
				Usage:       "Do not run a local agent and register a local kubelet",
				Hidden:      true,
				Destination: &ServerConfig.DisableAgent,
			},
			cli.StringSliceFlag{
				Hidden: true,
				Name:   "kube-controller-arg",
				Usage:  "(flags) Customized flag for kube-controller-manager process",
				Value:  &ServerConfig.ExtraControllerArgs,
			},
			cli.StringSliceFlag{
				Hidden: true,
				Name:   "kube-cloud-controller-arg",
				Usage:  "(flags) Customized flag for kube-cloud-controller-manager process",
				Value:  &ServerConfig.ExtraCloudControllerArgs,
			},
		},
	}
}
<|endoftext|>


<commit_before>package pigae

import (
	"fmt"
	"math/rand"
	"net/http"
	"net/url"
	"time"

	"github.com/gorilla/mux"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
)

// ThemeVersion is the version of the current CSS-JS theme.
// It is the name of the folder containing the theme files.
const ThemeVersion = "default"

// ThemeDate is the prefix used for "revving" the static files and enable long-term HTTP cache.
// It MUST end with underscore _ (see app.yaml)
const ThemeDate = "20171211_"

var r = mux.NewRouter()

func init() {
	initEnv()
	initToggles()
	initRoutes()

	// We want the random results to be different even if we reboot the server. Thus, we use
	// the clock to seed the default generator.
	// See https://www.programming-idioms.org/idiom/70/use-clock-as-random-generator-seed/346/go
	rand.Seed(time.Now().UnixNano())
}

func initRoutes() {
	if !toggles["online"] {
		handle("/", makeWall(" Under maintenance."))
		//r.HandleFunc("/", makeWall(" Coming soon."))
	} else {
		//handle("/", makeWall(" Coming soon."))
		handle("/", home)
		handle("/home", home)
		handle("/wall", makeWall(" Coming soon."))
		handle("/about", about)
		handle("/idiom/{idiomId}", idiomDetail)
		handle("/idiom/{idiomId}/impl/{implId}", idiomDetail)
		handle("/idiom/{idiomId}/{idiomTitle}", idiomDetail)
		handle("/idiom/{idiomId}/diff/{v1}/{v2}", versionDiff)
		handle("/idiom/{idiomId}/{idiomTitle}/{implId}/{implLang}", idiomDetail)
		handle("/history/{idiomId}", idiomHistory)
		handle("/revert", revertIdiomVersion)
		handle("/history-restore", restoreIdiomVersion)
		handle("/all-idioms", allIdioms)
		handle("/random-idiom/having/{havingLang}", randomIdiomHaving)
		handle("/random-idiom/not-having/{notHavingLang}", randomIdiomNotHaving)
		handle("/random-idiom", randomIdiom)
		handle("/search", searchRedirect)
		handle("/search/{q}", search)
		handle("/list-by-language/{langs}", listByLanguage)
		handle("/missing-fields/{lang}", missingList)
		handle("/idiom-picture", idiomPicture)
		handle("/rss-recently-created", rssRecentlyCreated)
		handle("/rss-recently-updated", rssRecentlyUpdated)
		handle("/my/{nickname}/{langs}", bookmarkableUserURL)
		handle("/my/{langs}", bookmarkableUserURL)
		handle("/cheatsheet/{lang}", cheatsheet)
		handleAjax("/typeahead-languages", typeaheadLanguages)
		handleAjax("/ajax-other-implementations", ajaxOtherImplementations)
		if toggles["writable"] {
			// When not in "read-only" mode
			handle("/idiom-save", idiomSave)
			handle("/idiom-edit/{idiomId}", idiomEdit)
			handle("/idiom-add-picture/{idiomId}", idiomAddPicture)
			handle("/idiom-save-picture", idiomSavePicture)
			handle("/impl-edit/{idiomId}/{implId}", implEdit)
			//handle("/fake-idiom-save", fakeIdiomSave)
			handle("/idiom-create", idiomCreate)
			handle("/impl-create/{idiomId}", implCreate)
			handle("/impl-create/{idiomId}/{lang}", implCreate)
			handle("/impl-save", implSave)
			// Ajax
			handleAjax("/ajax-idiom-vote", ajaxIdiomVote)
			handleAjax("/ajax-impl-vote", ajaxImplVote)
			handleAjax("/ajax-demo-site-suggest", ajaxDemoSiteSuggest)
			handleAjax("/ajax-user-message-box", userMessageBoxAjax)
			handleAjax("/ajax-dismiss-user-message", dismissUserMessage)
			handle("/about-block-project", ajaxAboutProject)
			handle("/about-block-all-idioms", ajaxAboutAllIdioms)
			handle("/about-block-language-coverage", ajaxAboutLanguageCoverage)
			handle("/about-block-rss", ajaxAboutRss)
			handle("/about-block-cheatsheets", ajaxAboutCheatsheets)
			handle("/about-block-see-also", ajaxAboutSeeAlso)
			handle("/about-block-contact", ajaxAboutContact)
			// Admin
			handle("/admin", admin)
			handle("/admin-data-export", adminExport)
			handle("/admin-data-import", adminImport)
			handle("/admin-resave-entities", adminResaveEntities)
			handleAjax("/admin-repair-history-versions", adminRepairHistoryVersions)
			handleAjax("/admin-data-import-ajax", adminImportAjax)
			handleAjax("/admin-reindex-ajax", adminReindexAjax)
			handleAjax("/admin-refresh-toggles-ajax", ajaxRefreshToggles)
			handleAjax("/admin-set-toggle-ajax", ajaxSetToggle)
			handleAjax("/admin-create-relation-ajax", ajaxCreateRelation)
			handleAjax("/admin-idiom-delete", idiomDelete)
			handleAjax("/admin-impl-delete", implDelete)
			handleAjax("/admin-send-message-for-user", sendMessageForUserAjax)
		}

		handle("/auth", handleAuth)
		handle("/_ah/login_required", handleAuth)
	}
	http.Handle("/", r)
}

// Request will fail if path parameters are missing
var neededPathVariables = map[string][]string{
	"/idiom/{idiomId}":                                  {"idiomId"},
	"/idiom/{idiomId}/impl/{implId}":                    {"idiomId"},
	"/idiom/{idiomId}/{idiomTitle}":                     {"idiomId"},
	"/idiom/{idiomId}/{idiomTitle}/{implId}/{implLang}": {"idiomId"},
	"/search/{q}":                                       {"q"},
	"/my/{nickname}/{langs}":                            {"nickname", "langs"},
	"/idiom-edit/{idiomId}":                             {"idiomId"},
	"/idiom-add-picture/{idiomId}":                      {"idiomId"},
	"/impl-edit/{idiomId}/{implId}":                     {"idiomId", "implId"},
	"/impl-create/{idiomId}":                            {"idiomId"},
	"/impl-create/{idiomId}/{lang}":                     {"idiomId"},
	"/cheatsheet/{lang}":                                {"lang"},
}

// Request will fail if it doesn't provide the required GET or POST parameters
var neededParameters = map[string][]string{
	"/typeahead-languages":         { /*todo*/ },
	"/idiom-save":                  {"idiom_title"},
	"/idiom-save-picture":          { /*todo*/ },
	"/impl-save":                   {"idiom_id", "impl_code"},
	"/revert":                      {"idiomId", "version"},
	"/ajax-idiom-vote":             {"idiomId", "choice"},
	"/ajax-impl-vote":              {"implId", "choice"},
	"/ajax-demo-site-suggest":      { /*todo*/ },
	"/ajax-dismiss-user-message":   {"key"},
	"/admin-data-export":           { /*todo*/ },
	"/admin-data-import":           { /*todo*/ },
	"/admin-data-import-ajax":      { /*todo*/ },
	"/admin-set-toggle-ajax":       {"toggle", "value"},
	"/admin-create-relation-ajax":  {"idiomAId", "idiomBId"},
	"/admin-idiom-delete":          {"idiomId"},
	"/admin-impl-delete":           {"idiomId", "implId"},
	"/admin-send-message-for-user": {"username", "message"},
}

// Request will fail if corresponding toggle is off
var neededToggles = map[string][]string{
	"/home":                         {"online"},
	"/search":                       {"searchable"},
	"/search/{q}":                   {"searchable"},
	"/idiom-save":                   {"writable"},
	"/idiom-edit/{idiomId}":         {"writable", "writable", "idiomEditing"},
	"/idiom-add-picture/{idiomId}":  {"writable", "idiomEditing"},
	"/idiom-save-picture":           {"writable", "idiomEditing"},
	"/impl-edit/{idiomId}/{implId}": {"writable", "implEditing"},
	"/idiom-create":                 {"writable"},
	"/impl-create/{idiomId}":        {"writable", "implAddition"},
	"/impl-create/{idiomId}/{lang}": {"writable", "implAddition"},
	"/impl-save":                    {"writable"},
	"/ajax-idiom-vote":              {"writable"},
	"/ajax-impl-vote":               {"writable"},
	"/admin":                        {"administrable"},
	"/admin-data-export":            {"administrable"},
	"/admin-data-import":            {"administrable"},
	"/admin-data-import-ajax":       {"administrable"},
	"/admin-set-toggle-ajax":        {"administrable"},
	"/admin-create-relation-ajax":   {"administrable"},
	"/admin-idiom-delete":           {"administrable"},
	"/admin-impl-delete":            {"administrable"},
}

type standardHandler func(w http.ResponseWriter, r *http.Request)
type betterHandler func(w http.ResponseWriter, r *http.Request) error

// Wrap HandleFunc with
// - error handling
// - mandatory path variables check
// - mandatory parameters check
// - toggles check
func handle(path string, h betterHandler) {
	r.HandleFunc(path,
		func(w http.ResponseWriter, r *http.Request) {
			if isSpam(w, r) {
				return
			}

			defer func() {
				if msg := recover(); msg != nil {
					msgStr := fmt.Sprintf("%v", msg)
					errorPage(w, r, PiError{msgStr, http.StatusInternalServerError})
					return
				}
			}()
			if configTime == "0" {
				c := appengine.NewContext(r)
				_ = refreshToggles(c)
				// If it fails... well, ignore for now and continue with non-fresh toggles.
			}

			if err := muxVarsMissing(w, r, neededPathVariables[path]...); err != nil {
				errorPage(w, r, err)
				return
			}
			if err := togglesMissing(w, r, neededToggles[path]...); err != nil {
				errorPage(w, r, err)
				return
			}
			if err := parametersMissing(w, r, neededParameters[path]...); err != nil {
				errorPage(w, r, err)
				return
			}
			err := h(w, r)
			if err != nil {
				errorPage(w, r, err)
			}
		})
}

func handleAjax(path string, h betterHandler) {
	r.HandleFunc(path,
		func(w http.ResponseWriter, r *http.Request) {
			if isSpam(w, r) {
				return
			}

			defer func() {
				if msg := recover(); msg != nil {
					msgStr := fmt.Sprintf("%v", msg)
					errorJSON(w, r, PiError{msgStr, http.StatusInternalServerError})
					return
				}
			}()
			if configTime == "0" {
				c := appengine.NewContext(r)
				_ = refreshToggles(c)
				// If it fails... well, ignore for now and continue with non-fresh toggles.
			}

			if err := muxVarsMissing(w, r, neededPathVariables[path]...); err != nil {
				errorJSON(w, r, err)
				return
			}
			if err := togglesMissing(w, r, neededToggles[path]...); err != nil {
				errorJSON(w, r, err)
				return
			}
			if err := parametersMissing(w, r, neededParameters[path]...); err != nil {
				errorJSON(w, r, err)
				return
			}
			err := h(w, r)
			if err != nil {
				errorJSON(w, r, err)
			}
		})
}

var datastoreDao = GaeDatastoreAccessor{}
var memcachedDao = MemcacheDatastoreAccessor{datastoreDao}
var dao = memcachedDao

var daoVotes = GaeVotesAccessor{}

func parametersMissing(w http.ResponseWriter, r *http.Request, params ...string) error {
	missing := []string{}
	for _, param := range params {
		if r.FormValue(param) == "" {
			missing = append(missing, param)
		}
	}
	if len(missing) > 0 {
		return PiError{fmt.Sprintf("Missing parameters : %s", missing), http.StatusBadRequest}
	}
	return nil
}

// Looks in gorilla mux populated variables
func muxVarsMissing(w http.ResponseWriter, r *http.Request, params ...string) error {
	missing := []string{}
	muxvars := mux.Vars(r)
	for _, param := range params {
		if muxvars[param] == "" {
			missing = append(missing, param)
		}
	}
	if len(missing) > 0 {
		return PiError{fmt.Sprintf("Missing parameters : %s", missing), http.StatusBadRequest}
	}
	return nil
}

func validateURLFormat(urlStr string) error {
	u, err := url.Parse(urlStr)
	if err != nil {
		return err
	}
	if !u.IsAbs() {
		return fmt.Errorf("Requires an absolute URL")
	}
	return nil
}

func validateURLFormatOrEmpty(urlStr string) error {
	if urlStr == "" {
		return nil
	}
	return validateURLFormat(urlStr)
}

/*
func logIf(err error, logfunc func(format string, args ...interface{}), when string) {
	if err != nil {
		logfunc("Problem on %v: %v", when, err.Error())
	}
}
*/

func logIf(err error, logfunc func(c context.Context, format string, args ...interface{}), c context.Context, when string) {
	if err != nil {
		logfunc(c, "Problem on %v: %v", when, err.Error())
	}
}
<commit_msg>Updated static files.<commit_after>package pigae

import (
	"fmt"
	"math/rand"
	"net/http"
	"net/url"
	"time"

	"github.com/gorilla/mux"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
)

// ThemeVersion is the version of the current CSS-JS theme.
// It is the name of the folder containing the theme files.
const ThemeVersion = "default"

// ThemeDate is the prefix used for "revving" the static files and enable long-term HTTP cache.
// It MUST end with underscore _ (see app.yaml)
const ThemeDate = "20190303_"

var r = mux.NewRouter()

func init() {
	initEnv()
	initToggles()
	initRoutes()

	// We want the random results to be different even if we reboot the server. Thus, we use
	// the clock to seed the default generator.
	// See https://www.programming-idioms.org/idiom/70/use-clock-as-random-generator-seed/346/go
	rand.Seed(time.Now().UnixNano())
}

func initRoutes() {
	if !toggles["online"] {
		handle("/", makeWall(" Under maintenance."))
		//r.HandleFunc("/", makeWall(" Coming soon."))
	} else {
		//handle("/", makeWall(" Coming soon."))
		handle("/", home)
		handle("/home", home)
		handle("/wall", makeWall(" Coming soon."))
		handle("/about", about)
		handle("/idiom/{idiomId}", idiomDetail)
		handle("/idiom/{idiomId}/impl/{implId}", idiomDetail)
		handle("/idiom/{idiomId}/{idiomTitle}", idiomDetail)
		handle("/idiom/{idiomId}/diff/{v1}/{v2}", versionDiff)
		handle("/idiom/{idiomId}/{idiomTitle}/{implId}/{implLang}", idiomDetail)
		handle("/history/{idiomId}", idiomHistory)
		handle("/revert", revertIdiomVersion)
		handle("/history-restore", restoreIdiomVersion)
		handle("/all-idioms", allIdioms)
		handle("/random-idiom/having/{havingLang}", randomIdiomHaving)
		handle("/random-idiom/not-having/{notHavingLang}", randomIdiomNotHaving)
		handle("/random-idiom", randomIdiom)
		handle("/search", searchRedirect)
		handle("/search/{q}", search)
		handle("/list-by-language/{langs}", listByLanguage)
		handle("/missing-fields/{lang}", missingList)
		handle("/idiom-picture", idiomPicture)
		handle("/rss-recently-created", rssRecentlyCreated)
		handle("/rss-recently-updated", rssRecentlyUpdated)
		handle("/my/{nickname}/{langs}", bookmarkableUserURL)
		handle("/my/{langs}", bookmarkableUserURL)
		handle("/cheatsheet/{lang}", cheatsheet)
		handleAjax("/typeahead-languages", typeaheadLanguages)
		handleAjax("/ajax-other-implementations", ajaxOtherImplementations)
		if toggles["writable"] {
			// When not in "read-only" mode
			handle("/idiom-save", idiomSave)
			handle("/idiom-edit/{idiomId}", idiomEdit)
			handle("/idiom-add-picture/{idiomId}", idiomAddPicture)
			handle("/idiom-save-picture", idiomSavePicture)
			handle("/impl-edit/{idiomId}/{implId}", implEdit)
			//handle("/fake-idiom-save", fakeIdiomSave)
			handle("/idiom-create", idiomCreate)
			handle("/impl-create/{idiomId}", implCreate)
			handle("/impl-create/{idiomId}/{lang}", implCreate)
			handle("/impl-save", implSave)
			// Ajax
			handleAjax("/ajax-idiom-vote", ajaxIdiomVote)
			handleAjax("/ajax-impl-vote", ajaxImplVote)
			handleAjax("/ajax-demo-site-suggest", ajaxDemoSiteSuggest)
			handleAjax("/ajax-user-message-box", userMessageBoxAjax)
			handleAjax("/ajax-dismiss-user-message", dismissUserMessage)
			handle("/about-block-project", ajaxAboutProject)
			handle("/about-block-all-idioms", ajaxAboutAllIdioms)
			handle("/about-block-language-coverage", ajaxAboutLanguageCoverage)
			handle("/about-block-rss", ajaxAboutRss)
			handle("/about-block-cheatsheets", ajaxAboutCheatsheets)
			handle("/about-block-see-also", ajaxAboutSeeAlso)
			handle("/about-block-contact", ajaxAboutContact)
			// Admin
			handle("/admin", admin)
			handle("/admin-data-export", adminExport)
			handle("/admin-data-import", adminImport)
			handle("/admin-resave-entities", adminResaveEntities)
			handleAjax("/admin-repair-history-versions", adminRepairHistoryVersions)
			handleAjax("/admin-data-import-ajax", adminImportAjax)
			handleAjax("/admin-reindex-ajax", adminReindexAjax)
			handleAjax("/admin-refresh-toggles-ajax", ajaxRefreshToggles)
			handleAjax("/admin-set-toggle-ajax", ajaxSetToggle)
			handleAjax("/admin-create-relation-ajax", ajaxCreateRelation)
			handleAjax("/admin-idiom-delete", idiomDelete)
			handleAjax("/admin-impl-delete", implDelete)
			handleAjax("/admin-send-message-for-user", sendMessageForUserAjax)
		}

		handle("/auth", handleAuth)
		handle("/_ah/login_required", handleAuth)
	}
	http.Handle("/", r)
}

// Request will fail if path parameters are missing
var neededPathVariables = map[string][]string{
	"/idiom/{idiomId}":                                  {"idiomId"},
	"/idiom/{idiomId}/impl/{implId}":                    {"idiomId"},
	"/idiom/{idiomId}/{idiomTitle}":                     {"idiomId"},
	"/idiom/{idiomId}/{idiomTitle}/{implId}/{implLang}": {"idiomId"},
	"/search/{q}":                                       {"q"},
	"/my/{nickname}/{langs}":                            {"nickname", "langs"},
	"/idiom-edit/{idiomId}":                             {"idiomId"},
	"/idiom-add-picture/{idiomId}":                      {"idiomId"},
	"/impl-edit/{idiomId}/{implId}":                     {"idiomId", "implId"},
	"/impl-create/{idiomId}":                            {"idiomId"},
	"/impl-create/{idiomId}/{lang}":                     {"idiomId"},
	"/cheatsheet/{lang}":                                {"lang"},
}

// Request will fail if it doesn't provide the required GET or POST parameters
var neededParameters = map[string][]string{
	"/typeahead-languages":         { /*todo*/ },
	"/idiom-save":                  {"idiom_title"},
	"/idiom-save-picture":          { /*todo*/ },
	"/impl-save":                   {"idiom_id", "impl_code"},
	"/revert":                      {"idiomId", "version"},
	"/ajax-idiom-vote":             {"idiomId", "choice"},
	"/ajax-impl-vote":              {"implId", "choice"},
	"/ajax-demo-site-suggest":      { /*todo*/ },
	"/ajax-dismiss-user-message":   {"key"},
	"/admin-data-export":           { /*todo*/ },
	"/admin-data-import":           { /*todo*/ },
	"/admin-data-import-ajax":      { /*todo*/ },
	"/admin-set-toggle-ajax":       {"toggle", "value"},
	"/admin-create-relation-ajax":  {"idiomAId", "idiomBId"},
	"/admin-idiom-delete":          {"idiomId"},
	"/admin-impl-delete":           {"idiomId", "implId"},
	"/admin-send-message-for-user": {"username", "message"},
}

// Request will fail if corresponding toggle is off
var neededToggles = map[string][]string{
	"/home":                         {"online"},
	"/search":                       {"searchable"},
	"/search/{q}":                   {"searchable"},
	"/idiom-save":                   {"writable"},
	"/idiom-edit/{idiomId}":         {"writable", "writable", "idiomEditing"},
	"/idiom-add-picture/{idiomId}":  {"writable", "idiomEditing"},
	"/idiom-save-picture":           {"writable", "idiomEditing"},
	"/impl-edit/{idiomId}/{implId}": {"writable", "implEditing"},
	"/idiom-create":                 {"writable"},
	"/impl-create/{idiomId}":        {"writable", "implAddition"},
	"/impl-create/{idiomId}/{lang}": {"writable", "implAddition"},
	"/impl-save":                    {"writable"},
	"/ajax-idiom-vote":              {"writable"},
	"/ajax-impl-vote":               {"writable"},
	"/admin":                        {"administrable"},
	"/admin-data-export":            {"administrable"},
	"/admin-data-import":            {"administrable"},
	"/admin-data-import-ajax":       {"administrable"},
	"/admin-set-toggle-ajax":        {"administrable"},
	"/admin-create-relation-ajax":   {"administrable"},
	"/admin-idiom-delete":           {"administrable"},
	"/admin-impl-delete":            {"administrable"},
}

type standardHandler func(w http.ResponseWriter, r *http.Request)
type betterHandler func(w http.ResponseWriter, r *http.Request) error

// Wrap HandleFunc with
// - error handling
// - mandatory path variables check
// - mandatory parameters check
// - toggles check
func handle(path string, h betterHandler) {
	r.HandleFunc(path,
		func(w http.ResponseWriter, r *http.Request) {
			if isSpam(w, r) {
				return
			}

			defer func() {
				if msg := recover(); msg != nil {
					msgStr := fmt.Sprintf("%v", msg)
					errorPage(w, r, PiError{msgStr, http.StatusInternalServerError})
					return
				}
			}()
			if configTime == "0" {
				c := appengine.NewContext(r)
				_ = refreshToggles(c)
				// If it fails... well, ignore for now and continue with non-fresh toggles.
			}

			if err := muxVarsMissing(w, r, neededPathVariables[path]...); err != nil {
				errorPage(w, r, err)
				return
			}
			if err := togglesMissing(w, r, neededToggles[path]...); err != nil {
				errorPage(w, r, err)
				return
			}
			if err := parametersMissing(w, r, neededParameters[path]...); err != nil {
				errorPage(w, r, err)
				return
			}
			err := h(w, r)
			if err != nil {
				errorPage(w, r, err)
			}
		})
}

func handleAjax(path string, h betterHandler) {
	r.HandleFunc(path,
		func(w http.ResponseWriter, r *http.Request) {
			if isSpam(w, r) {
				return
			}

			defer func() {
				if msg := recover(); msg != nil {
					msgStr := fmt.Sprintf("%v", msg)
					errorJSON(w, r, PiError{msgStr, http.StatusInternalServerError})
					return
				}
			}()
			if configTime == "0" {
				c := appengine.NewContext(r)
				_ = refreshToggles(c)
				// If it fails... well, ignore for now and continue with non-fresh toggles.
			}

			if err := muxVarsMissing(w, r, neededPathVariables[path]...); err != nil {
				errorJSON(w, r, err)
				return
			}
			if err := togglesMissing(w, r, neededToggles[path]...); err != nil {
				errorJSON(w, r, err)
				return
			}
			if err := parametersMissing(w, r, neededParameters[path]...); err != nil {
				errorJSON(w, r, err)
				return
			}
			err := h(w, r)
			if err != nil {
				errorJSON(w, r, err)
			}
		})
}

var datastoreDao = GaeDatastoreAccessor{}
var memcachedDao = MemcacheDatastoreAccessor{datastoreDao}
var dao = memcachedDao

var daoVotes = GaeVotesAccessor{}

func parametersMissing(w http.ResponseWriter, r *http.Request, params ...string) error {
	missing := []string{}
	for _, param := range params {
		if r.FormValue(param) == "" {
			missing = append(missing, param)
		}
	}
	if len(missing) > 0 {
		return PiError{fmt.Sprintf("Missing parameters : %s", missing), http.StatusBadRequest}
	}
	return nil
}

// Looks in gorilla mux populated variables
func muxVarsMissing(w http.ResponseWriter, r *http.Request, params ...string) error {
	missing := []string{}
	muxvars := mux.Vars(r)
	for _, param := range params {
		if muxvars[param] == "" {
			missing = append(missing, param)
		}
	}
	if len(missing) > 0 {
		return PiError{fmt.Sprintf("Missing parameters : %s", missing), http.StatusBadRequest}
	}
	return nil
}

func validateURLFormat(urlStr string) error {
	u, err := url.Parse(urlStr)
	if err != nil {
		return err
	}
	if !u.IsAbs() {
		return fmt.Errorf("Requires an absolute URL")
	}
	return nil
}

func validateURLFormatOrEmpty(urlStr string) error {
	if urlStr == "" {
		return nil
	}
	return validateURLFormat(urlStr)
}

/*
func logIf(err error, logfunc func(format string, args ...interface{}), when string) {
	if err != nil {
		logfunc("Problem on %v: %v", when, err.Error())
	}
}
*/

func logIf(err error, logfunc func(c context.Context, format string, args ...interface{}), c context.Context, when string) {
	if err != nil {
		logfunc(c, "Problem on %v: %v", when, err.Error())
	}
}
<|endoftext|>