<commit_before>package config

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	jwt "github.com/dgrijalva/jwt-go"
	konfig "github.com/koding/kite/config"
	"github.com/koding/kite/kitekey"
)

// KonfigCache is a default konfig.bolt configuration.
//
// The konfig.bolt stores user configuration for
// KD / Klient apps.
var KonfigCache = &CacheOptions{
	File: filepath.Join(KodingHome(), "konfig.bolt"),
	BoltDB: &bolt.Options{
		Timeout: 5 * time.Second,
	},
	Bucket: []byte("konfig"),
}

// Endpoints represents a configuration of Koding
// endpoints, which are used by KD / Klient.
type Endpoints struct {
	// Koding base endpoint.
	Koding *Endpoint `json:"koding,omitempty"`

	// Tunnel / proxy environment endpoints.
	Tunnel  *Endpoint `json:"tunnel,omitempty"`
	IP      *Endpoint `json:"ip,omitempty"`
	IPCheck *Endpoint `json:"ipCheck,omitempty"`

	// Klient / KD endpoints.
	KlientLatest *Endpoint `json:"klientLatest,omitempty"`
	KDLatest     *Endpoint `json:"kdLatest,omitempty"`
	Klient       *Endpoint `json:"klient,omitempty"`
}

// Kloud gives an endpoint for Kloud kite.
func (e *Endpoints) Kloud() *Endpoint {
	return e.Koding.WithPath("/kloud/kite")
}

// Kontrol gives an endpoint for Kontrol kite.
func (e *Endpoints) Kontrol() *Endpoint {
	return e.Koding.WithPath("/kontrol/kite")
}

// Remote gives an endpoint for remote.api.
func (e *Endpoints) Remote() *Endpoint {
	return e.Koding.WithPath("/remote.api")
}

// Social gives an endpoint for social.api.
func (e *Endpoints) Social() *Endpoint {
	return e.Koding.WithPath("/api/social")
}

// Mount describes configuration of mounts.
type Mount struct {
	// Mount is a default home path of mounted directories.
	//
	// If empty, defaults to ~/koding/mnt/.
	Home string `json:"home,omitempty"`

	// Exports maps named mounts to local paths.
	//
	// The Exports["default"] export is used as
	// a default one when caller does not specify
	// a mount path.
	//
	// The Exports["default"] defaults to $HOME.
	Exports map[string]string `json:"exports,omitempty"`

	// Inspect configures behavior of mount inspect command.
	Inspect *MountInspect `json:"inspect,omitempty"`

	// Sync configures behavior of synchronization goroutines.
	Sync *MountSync `json:"sync,omitempty"`

	// Debug is a debug level used for logging within
	// mounts.
	//
	//   <=0  - turns off debug logging
	//   1    - turns on debug logging for syncer events
	//   2-8  - reserved for future use
	//   >=9  - turns on debug logging for fuse events
	//
	Debug int `json:"debug,omitempty"`
}

// MountInspect describes configuration of mount inspect command.
type MountInspect struct {
	// History configures the length of inspect history,
	// which is 100 by default.
	History int `json:"history,omitempty"`
}

// MountSync describes configuration of synchronization goroutines.
type MountSync struct {
	// Workers configures number of concurrent rsync processes,
	// which is 2 * cpu by default.
	Workers int `json:"workers,omitempty"`
}

// Export gives a path for the named mount.
//
// If the named mount does not exist, it returns false.
//
// If the path contains '~' - it is expended to
// a home directory of a current user.
func (m *Mount) Export(name string) (string, bool) {
	if m == nil {
		return "", false
	}

	if dir, ok := m.Exports[name]; ok {
		return expandHome(dir), true
	}

	return "", false
}

// Template represents a KD template configuration.
type Template struct {
	File string `json:"file,omitempty"`
}

// Konfig represents a single configuration stored
// in a konfig.bolt database.
type Konfig struct {
	Endpoints *Endpoints `json:"endpoints,omitempty"`

	KontrolURL string `json:"kontrolURL,omitempty"` // deprecated / read-only
	TunnelURL  string `json:"tunnelURL,omitempty"`  // deprecated / read-only

	// Kite configuration.
	Environment string `json:"environment,omitempty"`
	KiteKeyFile string `json:"kiteKeyFile,omitempty"`
	KiteKey     string `json:"kiteKey,omitempty"`

	// Mount describes configuration of mounts.
	Mount *Mount `json:"local,omitempty"`

	// Template describes configuration of KD template.
	Template *Template `json:"template,omitempty"`

	// Koding networking configuration.
	//
	// TODO(rjeczalik): store command line flags in konfig.bolt
	// per Koding executable (KD / Klient).
	TunnelID string `json:"tunnelID,omitempty"`

	// Public S3 bucket for writing logs.
	PublicBucketName   string `json:"publicBucketName,omitempty"`
	PublicBucketRegion string `json:"publicBucketRegion,omitempty"`

	Debug bool `json:"debug,string,omitempty"`
}

// KiteHome gives directory of the kite.key file.
//
// Deprecated: Use KiteKeyFile instead.
func (k *Konfig) KiteHome() string {
	return filepath.Dir(k.KiteKeyFile)
}

// KiteConfig build *kite.Config value which is used
// to initialize new kite.Kite values.
func (k *Konfig) KiteConfig() *konfig.Config {
	return k.buildKiteConfig()
}

// KlientGzURL gives an URL for Klient binary.
//
// TODO(rjeczalik): Rework into lookup and move to kloud/metadata package.
func (k *Konfig) KlientGzURL() string {
	u := *k.Endpoints.KlientLatest.Public.URL
	u.Path = path.Join(path.Dir(u.Path), "latest", "klient.gz")
	return u.String()
}

// Valid implements the stack.Validator interface.
func (k *Konfig) Valid() error {
	// TODO(rjeczalik): remove when KontrolURL is gone
	if _, err := url.Parse(k.KontrolURL); err == nil && k.KontrolURL != "" {
		return nil
	}

	if k.Endpoints == nil {
		return errors.New("endpoints are nil")
	}
	if k.Endpoints.Koding == nil {
		return errors.New("koding base endpoint is nil")
	}
	if k.Endpoints.Koding.Public == nil {
		return errors.New("public URL for koding base endpoint is nil")
	}
	return nil
}

// ID gives an identifier of the Konfig value.
//
// The konfig.bolt stores multiple configurations, one for each
// baseurl.
//
// The ID is unique per baseurl.
func (k *Konfig) ID() string {
	return ID(k.KodingPublic().String())
}

// ID creates an identifier for the given Koding base URL.
func ID(kodingURL string) string {
	if kodingURL == "" {
		return ""
	}
	if u, err := url.Parse(kodingURL); err == nil {
		// Since id is input sensitive we clean the path so "example.com/koding"
		// "example.com/koding/" are effectively the same urls.
		u.Path = strings.TrimRight(path.Clean(u.Path), "/")
		switch u.Scheme {
		case "http":
			u.Host = strings.TrimSuffix(u.Host, ":80")
		case "https":
			u.Host = strings.TrimSuffix(u.Host, ":443")
		}
		kodingURL = u.String()
	}
	hash := sha1.Sum([]byte(kodingURL))
	return hex.EncodeToString(hash[:4])
}

// KodingPublic is here for backward-compatibility purposes.
//
// Old klient and kd deployments may not have .Endpoints configuration
// on a first run, this is why we fallback to old KontrolURL field.
//
// Which we should eventually get rid of.
//
// Deprecated: Use k.Endpoints.Koding.Public instead.
func (k *Konfig) KodingPublic() *url.URL {
	if e := k.Endpoints; e != nil && e.Koding != nil && e.Koding.Public != nil {
		return e.Koding.Public.URL
	}

	if u, err := url.Parse(k.KontrolURL); err == nil {
		u.Path = ""
		return u
	}

	return nil
}

func (k *Konfig) buildKiteConfig() *konfig.Config {
	kiteKey := k.KiteKey

	if kiteKey == "" && k.KiteKeyFile != "" {
		if p, err := ioutil.ReadFile(k.KiteKeyFile); err == nil && len(p) != 0 {
			kiteKey = string(bytes.TrimSpace(p))
		}
	}

	if kiteKey != "" {
		tok, err := jwt.ParseWithClaims(kiteKey, &kitekey.KiteClaims{}, kitekey.GetKontrolKey)
		if err == nil {
			cfg := NewKiteConfig(k.Debug)

			if err = cfg.ReadToken(tok); err == nil {
				return cfg
			}
		}
	}

	if cfg, err := ReadKiteConfig(k.Debug); err == nil {
		return cfg
	}

	return NewKiteConfig(k.Debug)
}

// NewKonfigURL gives new configuration for the given base URL.
func NewKonfigURL(koding *url.URL) *Konfig {
	return &Konfig{
		Endpoints: &Endpoints{
			Koding: NewEndpointURL(koding),
		},
	}
}

type Konfigs map[string]*Konfig

func (kfg Konfigs) Slice() []*Konfig {
	keys := make([]string, 0, len(kfg))
	for k := range kfg {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	slice := make([]*Konfig, 0, len(kfg))
	for _, key := range keys {
		slice = append(slice, kfg[key])
	}
	return slice
}

// Environment is a hacky workaround for kd <-> klient environments.
// The managed klient expects to have kd from production channel,
// and devmanaged klient - from development. Depending from which
// app we load the the default Koding configuration, we need
// to cross-map the environments.
//
// TODO(rjeczalik): This should be fixed by removing managed / devmanaged
// channels - that is to improve "connect your vm" modal to
// not depend on special klient environment.
type Environments struct {
	Env       string
	KlientEnv string // Env is used if empty
	KDEnv     string // Env is used if empty
}

func (e *Environments) klientEnv() string {
	if e.KlientEnv != "" {
		return e.KlientEnv
	}
	return e.Env
}

func (e *Environments) kdEnv() string {
	if e.KDEnv != "" {
		return e.KDEnv
	}
	return e.Env
}

// NewKonfig creates new configuration by reading
// embedded kites/config/config.json file.
func NewKonfig(e *Environments) *Konfig {
	return &Konfig{
		Environment: e.Env,
		Endpoints: &Endpoints{
			Koding:       Builtin.Endpoints.KodingBase.Copy(),
			Tunnel:       Builtin.Endpoints.TunnelServer.Copy(),
			IP:           Builtin.Endpoints.IP.Copy(),
			IPCheck:      Builtin.Endpoints.IPCheck.Copy(),
			KlientLatest: ReplaceEnv(Builtin.Endpoints.KlientLatest, e.klientEnv()),
			KDLatest:     ReplaceEnv(Builtin.Endpoints.KDLatest, RmManaged(e.kdEnv())),
			Klient: &Endpoint{
				Private: &URL{&url.URL{
					Scheme: "http",
					Host:   "127.0.0.1:56789",
					Path:   "/kite",
				}},
			},
		},
		Mount: &Mount{
			Home: filepath.Join(CurrentUser.HomeDir, "koding", "mnt"),
			Exports: map[string]string{
				"default": CurrentUser.HomeDir,
			},
			Inspect: &MountInspect{
				History: 100,
			},
			Sync: &MountSync{
				Workers: 2 * runtime.NumCPU(),
			},
		},
		Template: &Template{
			File: "kd.yaml",
		},
		PublicBucketName:   Builtin.Buckets.PublicLogs.Name,
		PublicBucketRegion: Builtin.Buckets.PublicLogs.Region,
		Debug:              false,
	}
}

func expandHome(path string) string {
	const home = "~" + string(os.PathSeparator)

	switch {
	case path == "~":
		return CurrentUser.HomeDir
	case strings.HasPrefix(path, home):
		return filepath.Join(CurrentUser.HomeDir, path[len(home):])
	default:
		return path
	}
}
<commit_msg>kites/config: fix mount tag<commit_after>package config

import (
	"bytes"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/boltdb/bolt"
	jwt "github.com/dgrijalva/jwt-go"
	konfig "github.com/koding/kite/config"
	"github.com/koding/kite/kitekey"
)

// KonfigCache is a default konfig.bolt configuration.
//
// The konfig.bolt stores user configuration for
// KD / Klient apps.
var KonfigCache = &CacheOptions{
	File: filepath.Join(KodingHome(), "konfig.bolt"),
	BoltDB: &bolt.Options{
		Timeout: 5 * time.Second,
	},
	Bucket: []byte("konfig"),
}

// Endpoints represents a configuration of Koding
// endpoints, which are used by KD / Klient.
type Endpoints struct {
	// Koding base endpoint.
	Koding *Endpoint `json:"koding,omitempty"`

	// Tunnel / proxy environment endpoints.
	Tunnel  *Endpoint `json:"tunnel,omitempty"`
	IP      *Endpoint `json:"ip,omitempty"`
	IPCheck *Endpoint `json:"ipCheck,omitempty"`

	// Klient / KD endpoints.
	KlientLatest *Endpoint `json:"klientLatest,omitempty"`
	KDLatest     *Endpoint `json:"kdLatest,omitempty"`
	Klient       *Endpoint `json:"klient,omitempty"`
}

// Kloud gives an endpoint for Kloud kite.
func (e *Endpoints) Kloud() *Endpoint {
	return e.Koding.WithPath("/kloud/kite")
}

// Kontrol gives an endpoint for Kontrol kite.
func (e *Endpoints) Kontrol() *Endpoint {
	return e.Koding.WithPath("/kontrol/kite")
}

// Remote gives an endpoint for remote.api.
func (e *Endpoints) Remote() *Endpoint {
	return e.Koding.WithPath("/remote.api")
}

// Social gives an endpoint for social.api.
func (e *Endpoints) Social() *Endpoint {
	return e.Koding.WithPath("/api/social")
}

// Mount describes configuration of mounts.
type Mount struct {
	// Mount is a default home path of mounted directories.
	//
	// If empty, defaults to ~/koding/mnt/.
	Home string `json:"home,omitempty"`

	// Exports maps named mounts to local paths.
	//
	// The Exports["default"] export is used as
	// a default one when caller does not specify
	// a mount path.
	//
	// The Exports["default"] defaults to $HOME.
	Exports map[string]string `json:"exports,omitempty"`

	// Inspect configures behavior of mount inspect command.
	Inspect *MountInspect `json:"inspect,omitempty"`

	// Sync configures behavior of synchronization goroutines.
	Sync *MountSync `json:"sync,omitempty"`

	// Debug is a debug level used for logging within
	// mounts.
	//
	//   <=0  - turns off debug logging
	//   1    - turns on debug logging for syncer events
	//   2-8  - reserved for future use
	//   >=9  - turns on debug logging for fuse events
	//
	Debug int `json:"debug,omitempty"`
}

// MountInspect describes configuration of mount inspect command.
type MountInspect struct {
	// History configures the length of inspect history,
	// which is 100 by default.
	History int `json:"history,omitempty"`
}

// MountSync describes configuration of synchronization goroutines.
type MountSync struct {
	// Workers configures number of concurrent rsync processes,
	// which is 2 * cpu by default.
	Workers int `json:"workers,omitempty"`
}

// Export gives a path for the named mount.
//
// If the named mount does not exist, it returns false.
//
// If the path contains '~' - it is expended to
// a home directory of a current user.
func (m *Mount) Export(name string) (string, bool) {
	if m == nil {
		return "", false
	}

	if dir, ok := m.Exports[name]; ok {
		return expandHome(dir), true
	}

	return "", false
}

// Template represents a KD template configuration.
type Template struct {
	File string `json:"file,omitempty"`
}

// Konfig represents a single configuration stored
// in a konfig.bolt database.
type Konfig struct {
	Endpoints *Endpoints `json:"endpoints,omitempty"`

	KontrolURL string `json:"kontrolURL,omitempty"` // deprecated / read-only
	TunnelURL  string `json:"tunnelURL,omitempty"`  // deprecated / read-only

	// Kite configuration.
	Environment string `json:"environment,omitempty"`
	KiteKeyFile string `json:"kiteKeyFile,omitempty"`
	KiteKey     string `json:"kiteKey,omitempty"`

	// Mount describes configuration of mounts.
	Mount *Mount `json:"mount,omitempty"`

	// Template describes configuration of KD template.
	Template *Template `json:"template,omitempty"`

	// Koding networking configuration.
	//
	// TODO(rjeczalik): store command line flags in konfig.bolt
	// per Koding executable (KD / Klient).
	TunnelID string `json:"tunnelID,omitempty"`

	// Public S3 bucket for writing logs.
	PublicBucketName   string `json:"publicBucketName,omitempty"`
	PublicBucketRegion string `json:"publicBucketRegion,omitempty"`

	Debug bool `json:"debug,string,omitempty"`
}

// KiteHome gives directory of the kite.key file.
//
// Deprecated: Use KiteKeyFile instead.
func (k *Konfig) KiteHome() string {
	return filepath.Dir(k.KiteKeyFile)
}

// KiteConfig build *kite.Config value which is used
// to initialize new kite.Kite values.
func (k *Konfig) KiteConfig() *konfig.Config {
	return k.buildKiteConfig()
}

// KlientGzURL gives an URL for Klient binary.
//
// TODO(rjeczalik): Rework into lookup and move to kloud/metadata package.
func (k *Konfig) KlientGzURL() string {
	u := *k.Endpoints.KlientLatest.Public.URL
	u.Path = path.Join(path.Dir(u.Path), "latest", "klient.gz")
	return u.String()
}

// Valid implements the stack.Validator interface.
func (k *Konfig) Valid() error {
	// TODO(rjeczalik): remove when KontrolURL is gone
	if _, err := url.Parse(k.KontrolURL); err == nil && k.KontrolURL != "" {
		return nil
	}

	if k.Endpoints == nil {
		return errors.New("endpoints are nil")
	}
	if k.Endpoints.Koding == nil {
		return errors.New("koding base endpoint is nil")
	}
	if k.Endpoints.Koding.Public == nil {
		return errors.New("public URL for koding base endpoint is nil")
	}
	return nil
}

// ID gives an identifier of the Konfig value.
//
// The konfig.bolt stores multiple configurations, one for each
// baseurl.
//
// The ID is unique per baseurl.
func (k *Konfig) ID() string {
	return ID(k.KodingPublic().String())
}

// ID creates an identifier for the given Koding base URL.
func ID(kodingURL string) string {
	if kodingURL == "" {
		return ""
	}
	if u, err := url.Parse(kodingURL); err == nil {
		// Since id is input sensitive we clean the path so "example.com/koding"
		// "example.com/koding/" are effectively the same urls.
		u.Path = strings.TrimRight(path.Clean(u.Path), "/")
		switch u.Scheme {
		case "http":
			u.Host = strings.TrimSuffix(u.Host, ":80")
		case "https":
			u.Host = strings.TrimSuffix(u.Host, ":443")
		}
		kodingURL = u.String()
	}
	hash := sha1.Sum([]byte(kodingURL))
	return hex.EncodeToString(hash[:4])
}

// KodingPublic is here for backward-compatibility purposes.
//
// Old klient and kd deployments may not have .Endpoints configuration
// on a first run, this is why we fallback to old KontrolURL field.
//
// Which we should eventually get rid of.
//
// Deprecated: Use k.Endpoints.Koding.Public instead.
func (k *Konfig) KodingPublic() *url.URL {
	if e := k.Endpoints; e != nil && e.Koding != nil && e.Koding.Public != nil {
		return e.Koding.Public.URL
	}

	if u, err := url.Parse(k.KontrolURL); err == nil {
		u.Path = ""
		return u
	}

	return nil
}

func (k *Konfig) buildKiteConfig() *konfig.Config {
	kiteKey := k.KiteKey

	if kiteKey == "" && k.KiteKeyFile != "" {
		if p, err := ioutil.ReadFile(k.KiteKeyFile); err == nil && len(p) != 0 {
			kiteKey = string(bytes.TrimSpace(p))
		}
	}

	if kiteKey != "" {
		tok, err := jwt.ParseWithClaims(kiteKey, &kitekey.KiteClaims{}, kitekey.GetKontrolKey)
		if err == nil {
			cfg := NewKiteConfig(k.Debug)

			if err = cfg.ReadToken(tok); err == nil {
				return cfg
			}
		}
	}

	if cfg, err := ReadKiteConfig(k.Debug); err == nil {
		return cfg
	}

	return NewKiteConfig(k.Debug)
}

// NewKonfigURL gives new configuration for the given base URL.
func NewKonfigURL(koding *url.URL) *Konfig {
	return &Konfig{
		Endpoints: &Endpoints{
			Koding: NewEndpointURL(koding),
		},
	}
}

type Konfigs map[string]*Konfig

func (kfg Konfigs) Slice() []*Konfig {
	keys := make([]string, 0, len(kfg))
	for k := range kfg {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	slice := make([]*Konfig, 0, len(kfg))
	for _, key := range keys {
		slice = append(slice, kfg[key])
	}
	return slice
}

// Environment is a hacky workaround for kd <-> klient environments.
// The managed klient expects to have kd from production channel,
// and devmanaged klient - from development. Depending from which
// app we load the the default Koding configuration, we need
// to cross-map the environments.
//
// TODO(rjeczalik): This should be fixed by removing managed / devmanaged
// channels - that is to improve "connect your vm" modal to
// not depend on special klient environment.
type Environments struct {
	Env       string
	KlientEnv string // Env is used if empty
	KDEnv     string // Env is used if empty
}

func (e *Environments) klientEnv() string {
	if e.KlientEnv != "" {
		return e.KlientEnv
	}
	return e.Env
}

func (e *Environments) kdEnv() string {
	if e.KDEnv != "" {
		return e.KDEnv
	}
	return e.Env
}

// NewKonfig creates new configuration by reading
// embedded kites/config/config.json file.
func NewKonfig(e *Environments) *Konfig {
	return &Konfig{
		Environment: e.Env,
		Endpoints: &Endpoints{
			Koding:       Builtin.Endpoints.KodingBase.Copy(),
			Tunnel:       Builtin.Endpoints.TunnelServer.Copy(),
			IP:           Builtin.Endpoints.IP.Copy(),
			IPCheck:      Builtin.Endpoints.IPCheck.Copy(),
			KlientLatest: ReplaceEnv(Builtin.Endpoints.KlientLatest, e.klientEnv()),
			KDLatest:     ReplaceEnv(Builtin.Endpoints.KDLatest, RmManaged(e.kdEnv())),
			Klient: &Endpoint{
				Private: &URL{&url.URL{
					Scheme: "http",
					Host:   "127.0.0.1:56789",
					Path:   "/kite",
				}},
			},
		},
		Mount: &Mount{
			Home: filepath.Join(CurrentUser.HomeDir, "koding", "mnt"),
			Exports: map[string]string{
				"default": CurrentUser.HomeDir,
			},
			Inspect: &MountInspect{
				History: 100,
			},
			Sync: &MountSync{
				Workers: 2 * runtime.NumCPU(),
			},
		},
		Template: &Template{
			File: "kd.yaml",
		},
		PublicBucketName:   Builtin.Buckets.PublicLogs.Name,
		PublicBucketRegion: Builtin.Buckets.PublicLogs.Region,
		Debug:              false,
	}
}

func expandHome(path string) string {
	const home = "~" + string(os.PathSeparator)

	switch {
	case path == "~":
		return CurrentUser.HomeDir
	case strings.HasPrefix(path, home):
		return filepath.Join(CurrentUser.HomeDir, path[len(home):])
	default:
		return path
	}
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/codingconcepts/albert/pkg/orchestrator"
	nats "github.com/nats-io/go-nats"
)

// NatsProcessor performs a scatter-gather operation
// using NATS as the underlying message bus.
type NatsProcessor struct {
	conn           *nats.Conn
	gatherChanSize int
	gatherTimeout  time.Duration
}

// NewNatsProcessor returns the pointer to a new instance of a
// NatsProcessor.
func newNatsProcessor(conn *nats.Conn, config *orchestrator.Config) (p *NatsProcessor) {
	return &NatsProcessor{
		conn:           conn,
		gatherChanSize: config.GatherChanSize,
		gatherTimeout:  config.GatherTimeout.Duration,
	}
}

// Gather performs a "scatter gather" operation against
// an unknown number of Applications.
// See http://bit.ly/2oEiquY for more information.
func (p *NatsProcessor) Gather(application string) (msgs []string, err error) {
	msgs = []string{}
	responses := make(chan *nats.Msg, p.gatherChanSize)
	defer close(responses)

	reply := nats.NewInbox()
	sub, err := p.conn.ChanQueueSubscribe(reply, "", responses)
	if err != nil {
		return
	}
	defer sub.Unsubscribe()

	if err = p.conn.PublishRequest(application, reply, nil); err != nil {
		return
	}

	for {
		select {
		case <-time.After(p.gatherTimeout):
			return
		case msg := <-responses:
			msgs = append(msgs, msg.Reply)
		}
	}
}

// IssueKill publishes a kill command for a given Application
// and ApplicationType combination.
func (p *NatsProcessor) IssueKill(topic string) (err error) {
	return p.conn.Publish(topic, nil)
}
<commit_msg>Made NATS processor unexported<commit_after>package main

import (
	"time"

	"github.com/codingconcepts/albert/pkg/orchestrator"
	nats "github.com/nats-io/go-nats"
)

type natsProcessor struct {
	conn           *nats.Conn
	gatherChanSize int
	gatherTimeout  time.Duration
}

// NewNatsProcessor returns the pointer to a new instance of a
// NatsProcessor.
func newNatsProcessor(conn *nats.Conn, config *orchestrator.Config) (p *natsProcessor) {
	return &natsProcessor{
		conn:           conn,
		gatherChanSize: config.GatherChanSize,
		gatherTimeout:  config.GatherTimeout.Duration,
	}
}

// Gather performs a "scatter gather" operation against
// an unknown number of Applications.
// See http://bit.ly/2oEiquY for more information.
func (p *natsProcessor) Gather(application string) (msgs []string, err error) {
	msgs = []string{}
	responses := make(chan *nats.Msg, p.gatherChanSize)
	defer close(responses)

	reply := nats.NewInbox()
	sub, err := p.conn.ChanQueueSubscribe(reply, "", responses)
	if err != nil {
		return
	}
	defer sub.Unsubscribe()

	if err = p.conn.PublishRequest(application, reply, nil); err != nil {
		return
	}

	for {
		select {
		case <-time.After(p.gatherTimeout):
			return
		case msg := <-responses:
			msgs = append(msgs, msg.Reply)
		}
	}
}

// IssueKill publishes a kill command for a given Application
// and ApplicationType combination.
func (p *natsProcessor) IssueKill(topic string) (err error) {
	return p.conn.Publish(topic, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/urfave/cli"
)

func main() {

	app := cli.NewApp()
	app.Name = "zippy"
	app.Author = "Tugberk Ugurlu"
	app.Usage = "General purpose package distribution tool for your stuff (yes, all your stuff™)"

	app.Action = func(c *cli.Context) error {
		fmt.Print("hello and byee!\n")
		return cli.NewExitError("no pun intended", -1)
	}

	app.Run(os.Args)
}
<commit_msg>install command<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/urfave/cli"
)

func main() {

	app := cli.NewApp()
	app.Name = "zippy"
	app.Author = "Tugberk Ugurlu"
	app.Usage = "General purpose package distribution tool for your stuff (yes, all your stuff™)"

	app.Commands = []cli.Command{
		cli.Command{
			Name:  "install",
			Usage: "Downloads the specified package and outputs it onto the specified directory",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "version, v",
					Usage: "Version of the package to be installed",
				},

				cli.StringFlag{
					Name:  "outputFolder, o",
					Usage: "Path to the output folder for the package to be installed",
					Value: "./",
				},

				cli.StringFlag{
					Name:  "source, s",
					Usage: "Base URL for the desired zippy endpoint to be looked at for this install operation",
				},
			},
		},
	}

	app.Action = func(c *cli.Context) error {
		fmt.Print("hello and byee!\n")
		return cli.NewExitError("no pun intended", -1)
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package fetcher

import (
	"fmt"
	"os"
	"testing"

	"github.com/oinume/lekcije/server/model"
	"github.com/stretchr/testify/assert"
)

var _ = fmt.Print

func TestFetch(t *testing.T) {
	assert := assert.New(t)
	fetcher := NewTeacherLessonFetcher(nil, nil)
	teacher, _, err := fetcher.Fetch(1523)
	assert.NoError(err)
	assert.Equal("Crizelle", teacher.Name)
}

func TestParseHtml(t *testing.T) {
	assert := assert.New(t)
	fetcher := NewTeacherLessonFetcher(nil, nil)
	file, err := os.Open("testdata/5982.html")
	assert.NoError(err)
	defer file.Close()

	teacher, lessons, err := fetcher.parseHtml(model.NewTeacher(uint32(5982)), file)
	assert.Equal("Xai", teacher.Name)
	assert.True(len(lessons) > 0)
	for _, lesson := range lessons {
		if lesson.Status == "reservable" {

		}
	}
	//fmt.Printf("%v\n", spew.Sdump(lessons))
}

//予約可
<commit_msg>Improve parseHtml test<commit_after>package fetcher

import (
	"fmt"
	"os"
	"testing"

	"github.com/oinume/lekcije/server/model"
	"github.com/stretchr/testify/assert"
)

var _ = fmt.Print

func TestFetch(t *testing.T) {
	assert := assert.New(t)
	fetcher := NewTeacherLessonFetcher(nil, nil)
	teacher, _, err := fetcher.Fetch(1523)
	assert.NoError(err)
	assert.Equal("Crizelle", teacher.Name)
}

func TestParseHtml(t *testing.T) {
	assert := assert.New(t)
	fetcher := NewTeacherLessonFetcher(nil, nil)
	file, err := os.Open("testdata/5982.html")
	assert.NoError(err)
	defer file.Close()

	teacher, lessons, err := fetcher.parseHtml(model.NewTeacher(uint32(5982)), file)
	assert.Equal("Xai", teacher.Name)
	assert.True(len(lessons) > 0)
	for _, lesson := range lessons {
		if lesson.Datetime.Format("2006-01-02 15:04") == "2016-07-01 11:00" {
			assert.Equal("Finished", lesson.Status)
		}
		if lesson.Datetime.Format("2006-01-02 15:04") == "2016-07-01 16:30" {
			assert.Equal("Reservable", lesson.Status)
		}
		if lesson.Datetime.Format("2006-01-02 15:04") == "2016-07-01 18:00" {
			assert.Equal("Reserved", lesson.Status)
		}
	}
	//fmt.Printf("%v\n", spew.Sdump(lessons))
}

//予約可
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"github.com/jjyr/ringo"
	"github.com/jjyr/ringo/middleware"
)

type usersController struct {
	ringo.Controller
}

type User struct {
	Name string `json:"name" validate:"required"`
	Age  int    `json:"age"`
}

func (*usersController) List(c *ringo.Context) {
	c.String(200, "list users")
}

func (*usersController) Create(c *ringo.Context) {
	u := User{}
	if err := c.Bind(&u); err != nil {
		panic(err)
	}
	resp := struct {
		UserInfo *User
		Message  string `json:message`
	}{UserInfo: &u, Message: "hello"}
	c.JSON(200, resp)
}

func (*usersController) SayHehe(c *ringo.Context) {
	c.JSON(200, struct {
		Hello   string
		Message string
	}{Hello: "nihaoo",
		Message: "你好",
	})
}

func main() {
	flag.Parse()
	app := ringo.NewApp()
	r := ringo.NewRouter()
	r.GET("/hello", func(c *ringo.Context) {
		c.String(200, "hello world!")
	})

	app.Mount("/say", r)

	app.GET("/ping", func(c *ringo.Context) {
		log.Print("pong!")
		c.String(200, "pong!")
	})

	app.GET("/numbers/:n/echo/:n", func(c *ringo.Context) {
		log.Print(c.Request.URL.Query())
		log.Print(c.Params.ByName("n"))
	})

	u := usersController{}
	u.AddRoutes(ringo.ControllerRouteOption{Handler: "SayHehe", Method: "GET", Member: true, Path: "hehe"})
	app.AddController(&u, nil)

	app.Use(middleware.Recover())

	app.Run("localhost:8020")
}
<commit_msg>remove examples<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/msutter/go-pulp/pulp"
	"log"
	"time"
)

func main() {
	apiUser := "admin"
	apiPasswd := "admin"
	apiEndpoint := "pulp-lab-11"

	// create the client
	client := pulp.NewClient(apiEndpoint, apiUser, apiPasswd, nil)

	// repository options
	ro := &pulp.GetRepositoryOptions{
		Details: true,
	}

	repo := "test-repo-1-lab"

	// get the repo
	r, _, err := client.Repositories.GetRepository(repo, ro)
	fmt.Printf("%v\n", r)
	if err != nil {
		log.Fatal(err)
	}

	// get the repo
	callReport, _, err := client.Repositories.SyncRepository(repo)
	syncTaskId := callReport.SpawnedTasks[0].TaskId
	fmt.Printf("TaskId: %v\n", syncTaskId)
	if err != nil {
		log.Fatal(err)
	}

	state := "init"
	for state != "finished" {
		task, _, terr := client.Tasks.GetTask(syncTaskId)
		fmt.Printf("%v\n", task.RenderProgressStatus())

		state = task.State
		time.Sleep(500 * time.Millisecond)
		if terr != nil {
			log.Fatal(terr)
		}
	}
}
<commit_msg>remove obsolete func<commit_after>package main

import (
	"fmt"
	"github.com/msutter/go-pulp/pulp"
	"log"
	"time"
)

func main() {
	apiUser := "admin"
	apiPasswd := "admin"
	apiEndpoint := "pulp-lab-11"

	// create the client
	client := pulp.NewClient(apiEndpoint, apiUser, apiPasswd, nil)

	// repository options
	ro := &pulp.GetRepositoryOptions{
		Details: true,
	}

	repo := "test-repo-1-lab"

	// get the repo
	r, _, err := client.Repositories.GetRepository(repo, ro)
	fmt.Printf("%v\n", r)
	if err != nil {
		log.Fatal(err)
	}

	// get the repo
	callReport, _, err := client.Repositories.SyncRepository(repo)
	syncTaskId := callReport.SpawnedTasks[0].TaskId
	fmt.Printf("TaskId: %v\n", syncTaskId)
	if err != nil {
		log.Fatal(err)
	}

	state := "init"
	for state != "finished" {
		task, _, terr := client.Tasks.GetTask(syncTaskId)
		fmt.Printf("%v\n", task.State)

		state = task.State
		time.Sleep(500 * time.Millisecond)
		if terr != nil {
			log.Fatal(terr)
		}
	}
}
<|endoftext|>


<commit_before>package websocket

import (
	"encoding/json"
	"github.com/gorilla/websocket"
	"github.com/hackform/governor"
	"github.com/sirupsen/logrus"
	"net/http"
	"sync"
)

const (
	moduleID = "websocket"
)

type (
	// Websocket is a websocket management service
	Websocket interface {
		Subscribe(conn *Conn)
		Unsubscribe(conn *Conn)
		Broadcast(channel string, msg interface{}) *governor.Error
		NewConn(channels []string, w http.ResponseWriter, r *http.Request) (*Conn, *governor.Error)
	}

	websocketService struct {
		channels sync.Map
		upgrader *websocket.Upgrader
	}

	channel struct {
		id        string
		listeners map[*Conn]bool
	}

	// Conn describes a websocket connection subscribed to multiple channels
	Conn struct {
		w        Websocket
		conn     *websocket.Conn
		channels []string
	}
)

// New creates a new Websocket service
func New(conf governor.Config, l *logrus.Logger) Websocket {
	l.Info("initialized websocket service")

	return &websocketService{
		channels: sync.Map{},
		upgrader: &websocket.Upgrader{
			CheckOrigin: func(r *http.Request) bool {
        return true
    	},
		},
	}
}

func (w *websocketService) upsertChannel(s string) *channel {
	c := newChannel(s)
	a, _ := w.channels.LoadOrStore(s, c)
	return a.(*channel)
}

func (w *websocketService) getChannel(s string) (*channel, bool) {
	a, b := w.channels.Load(s)
	return a.(*channel), b
}

func (w *websocketService) removeChannel(s string) {
	w.channels.Delete(s)
}

// Subscribe adds a websocket connection to a channel
func (w *websocketService) Subscribe(conn *Conn) {
	for _, channel := range conn.channels {
		c := w.upsertChannel(channel)
		c.listeners[conn] = true
	}
}

// Unsubscribe removes a websocket connection from a channel
func (w *websocketService) Unsubscribe(conn *Conn) {
	for _, channel := range conn.channels {
		if c, ok := w.getChannel(channel); ok {
			delete(c.listeners, conn)
			if len(c.listeners) == 0 {
				// TODO: listeners may not be 0 here
				w.removeChannel(channel)
			}
		}
	}
}

const (
	moduleIDBroadcast = moduleID + ".Broadcast"
)

// Broadcast sends a message to all websocket connections in a channel
func (w *websocketService) Broadcast(channel string, msg interface{}) *governor.Error {
	c, validChannel := w.getChannel(channel)
	if !validChannel {
		return nil
	}
	m, err := json.Marshal(msg)
	if err != nil {
		return governor.NewError(moduleIDBroadcast, err.Error(), 0, http.StatusInternalServerError)
	}
	p, err := websocket.NewPreparedMessage(websocket.TextMessage, m)
	if err != nil {
		return governor.NewError(moduleIDBroadcast, err.Error(), 0, http.StatusInternalServerError)
	}
	for listener := range c.listeners {
		if err := listener.conn.WritePreparedMessage(p); err != nil {
			listener.Close()
		}
	}
	return nil
}

func newChannel(id string) *channel {
	return &channel{
		id:        id,
		listeners: make(map[*Conn]bool),
	}
}

const (
	moduleIDConn = moduleID + ".Conn"

	moduleIDConnNew = moduleIDConn + ".New"
)

// NewConn establishes a new websocket connection from an existing http connection
func (w *websocketService) NewConn(channels []string, rw http.ResponseWriter, r *http.Request) (*Conn, *governor.Error) {
	ws, err := w.upgrader.Upgrade(rw, r, nil)
	if err != nil {
		return nil, governor.NewErrorUser(moduleIDConnNew, err.Error(), 0, http.StatusBadRequest)
	}

	c := &Conn{
		w:        w,
		conn:     ws,
		channels: channels,
	}
	w.Subscribe(c)
	return c, nil
}

const (
	moduleIDConnRead = moduleIDConn + ".Read"
)

// Read reads a message to json
func (c *Conn) Read(msg interface{}) *governor.Error {
	if err := c.conn.ReadJSON(msg); err != nil {
		c.Close()
		return governor.NewError(moduleIDConnRead, err.Error(), 0, http.StatusInternalServerError)
	}
	return nil
}

// Close closes the websocket connection
func (c *Conn) Close() {
	c.w.Unsubscribe(c)
	if c.conn.Close() != nil {
	}
}
<commit_msg>Bugfix check if channel present in websocket map<commit_after>package websocket

import (
	"encoding/json"
	"github.com/gorilla/websocket"
	"github.com/hackform/governor"
	"github.com/sirupsen/logrus"
	"net/http"
	"sync"
)

const (
	moduleID = "websocket"
)

type (
	// Websocket is a websocket management service
	Websocket interface {
		Subscribe(conn *Conn)
		Unsubscribe(conn *Conn)
		Broadcast(channel string, msg interface{}) *governor.Error
		NewConn(channels []string, w http.ResponseWriter, r *http.Request) (*Conn, *governor.Error)
	}

	websocketService struct {
		channels sync.Map
		upgrader *websocket.Upgrader
	}

	channel struct {
		id        string
		listeners map[*Conn]bool
	}

	// Conn describes a websocket connection subscribed to multiple channels
	Conn struct {
		w        Websocket
		conn     *websocket.Conn
		channels []string
	}
)

// New creates a new Websocket service
func New(conf governor.Config, l *logrus.Logger) Websocket {
	l.Info("initialized websocket service")

	return &websocketService{
		channels: sync.Map{},
		upgrader: &websocket.Upgrader{
			CheckOrigin: func(r *http.Request) bool {
        return true
    	},
		},
	}
}

func (w *websocketService) upsertChannel(s string) *channel {
	c := newChannel(s)
	a, _ := w.channels.LoadOrStore(s, c)
	return a.(*channel)
}

func (w *websocketService) getChannel(s string) (*channel, bool) {
	a, b := w.channels.Load(s)
	if !b {
		return nil, false
	}
	return a.(*channel), true
}

func (w *websocketService) removeChannel(s string) {
	w.channels.Delete(s)
}

// Subscribe adds a websocket connection to a channel
func (w *websocketService) Subscribe(conn *Conn) {
	for _, channel := range conn.channels {
		c := w.upsertChannel(channel)
		c.listeners[conn] = true
	}
}

// Unsubscribe removes a websocket connection from a channel
func (w *websocketService) Unsubscribe(conn *Conn) {
	for _, channel := range conn.channels {
		if c, ok := w.getChannel(channel); ok {
			delete(c.listeners, conn)
			if len(c.listeners) == 0 {
				// TODO: listeners may not be 0 here
				w.removeChannel(channel)
			}
		}
	}
}

const (
	moduleIDBroadcast = moduleID + ".Broadcast"
)

// Broadcast sends a message to all websocket connections in a channel
func (w *websocketService) Broadcast(channel string, msg interface{}) *governor.Error {
	c, validChannel := w.getChannel(channel)
	if !validChannel {
		return nil
	}
	m, err := json.Marshal(msg)
	if err != nil {
		return governor.NewError(moduleIDBroadcast, err.Error(), 0, http.StatusInternalServerError)
	}
	p, err := websocket.NewPreparedMessage(websocket.TextMessage, m)
	if err != nil {
		return governor.NewError(moduleIDBroadcast, err.Error(), 0, http.StatusInternalServerError)
	}
	for listener := range c.listeners {
		if err := listener.conn.WritePreparedMessage(p); err != nil {
			listener.Close()
		}
	}
	return nil
}

func newChannel(id string) *channel {
	return &channel{
		id:        id,
		listeners: make(map[*Conn]bool),
	}
}

const (
	moduleIDConn = moduleID + ".Conn"

	moduleIDConnNew = moduleIDConn + ".New"
)

// NewConn establishes a new websocket connection from an existing http connection
func (w *websocketService) NewConn(channels []string, rw http.ResponseWriter, r *http.Request) (*Conn, *governor.Error) {
	ws, err := w.upgrader.Upgrade(rw, r, nil)
	if err != nil {
		return nil, governor.NewErrorUser(moduleIDConnNew, err.Error(), 0, http.StatusBadRequest)
	}

	c := &Conn{
		w:        w,
		conn:     ws,
		channels: channels,
	}
	w.Subscribe(c)
	return c, nil
}

const (
	moduleIDConnRead = moduleIDConn + ".Read"
)

// Read reads a message to json
func (c *Conn) Read(msg interface{}) *governor.Error {
	if err := c.conn.ReadJSON(msg); err != nil {
		c.Close()
		return governor.NewError(moduleIDConnRead, err.Error(), 0, http.StatusInternalServerError)
	}
	return nil
}

// Close closes the websocket connection
func (c *Conn) Close() {
	c.w.Unsubscribe(c)
	if c.conn.Close() != nil {
	}
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package main

import (
  "bufio"
  "log"
  "net"
  "strconv"
  "strings"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store *ArticleStore
  api *SRNdAPI
  database *Database
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  feeds map[NNTPConnection]bool
  infeed chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  feed := NNTPConnection{conn, bufio.NewReader(conn), inbound, self.debug, new(ConnectionInfo), policy, make(chan *NNTPMessage)}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return
  }
  defer self.listener.Close()

  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  self.syncAllFeeds()

  // start accepting incoming connections
  go self.mainloop()

  // start api
  go self.api.Mainloop()

  // loop over messages
  for {
    message := <- self.infeed
    // load message
    nntp := self.store.GetMessage(message, false)
    self.store.StoreArticle(nntp.Newsgroup, nntp.MessageID)
    // send to all outfeeds
    if nntp != nil {
      for feed , use := range self.feeds {
        if use {
          feed.send <- nntp
        }
      }
    }
  }
}

func (self *NNTPDaemon) syncAllFeeds() {
  var err error
  if self.sync_on_start {
    log.Println("sync all feeds")
    self.store.IterateAllArticles(func(messageID string) bool {
      msg := self.store.GetMessage(messageID, false)
      if msg != nil {
        for feed, use := range self.feeds {
            if use {
              err = feed.SendMessage(msg, self)
            }
        }
      }
      return false
    })
    log.Println("sync done")
  }

}

func (self *NNTPDaemon) mainloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunConnection(nntp)
  }
}

func (self *NNTPDaemon) RunConnection(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Println("SRNd NNTPD bound at", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {
  CheckConfig()
  log.Println("load config")
  self.conf = ReadConf()
  if self.conf == nil {
    log.Println("cannot load config")
    return false
  }
  self.infeed = make(chan string, 200)
  self.feeds = make(map[NNTPConnection]bool)

  self.database = new(Database)

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  err := self.database.Init(db_host, db_port, db_user, db_passwd)
  if err != nil {
    log.Println("failed to initialize database", err)
    return false
  }

  self.store = new(ArticleStore)
  self.store.directory = self.conf.store["base_dir"]
  self.store.Init()

  self.api = new(SRNdAPI)
  self.api.Init(self)

  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  if self.sync_on_start {
    log.Println("sync on start")
  }
  self.bind_addr = self.conf.daemon["bind"]
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  return true
}
<commit_msg>add sleep between reconnects so stuff does not spam (more)<commit_after>//
// daemon.go
//
package main

import (
  "bufio"
  "log"
  "net"
  "strconv"
  "strings"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store *ArticleStore
  api *SRNdAPI
  database *Database
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  feeds map[NNTPConnection]bool
  infeed chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  feed := NNTPConnection{conn, bufio.NewReader(conn), inbound, self.debug, new(ConnectionInfo), policy, make(chan *NNTPMessage)}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return
  }
  defer self.listener.Close()

  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  self.syncAllFeeds()

  // start accepting incoming connections
  go self.mainloop()

  // start api
  go self.api.Mainloop()

  // loop over messages
  for {
    message := <- self.infeed
    // load message
    nntp := self.store.GetMessage(message, false)
    self.store.StoreArticle(nntp.Newsgroup, nntp.MessageID)
    // send to all outfeeds
    if nntp != nil {
      for feed , use := range self.feeds {
        if use {
          feed.send <- nntp
        }
      }
    }
  }
}

func (self *NNTPDaemon) syncAllFeeds() {
  var err error
  if self.sync_on_start {
    log.Println("sync all feeds")
    self.store.IterateAllArticles(func(messageID string) bool {
      msg := self.store.GetMessage(messageID, false)
      if msg != nil {
        for feed, use := range self.feeds {
            if use {
              err = feed.SendMessage(msg, self)
            }
        }
      }
      return false
    })
    log.Println("sync done")
  }

}

func (self *NNTPDaemon) mainloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunConnection(nntp)
  }
}

func (self *NNTPDaemon) RunConnection(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Println("SRNd NNTPD bound at", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {
  CheckConfig()
  log.Println("load config")
  self.conf = ReadConf()
  if self.conf == nil {
    log.Println("cannot load config")
    return false
  }
  self.infeed = make(chan string, 200)
  self.feeds = make(map[NNTPConnection]bool)

  self.database = new(Database)

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  err := self.database.Init(db_host, db_port, db_user, db_passwd)
  if err != nil {
    log.Println("failed to initialize database", err)
    return false
  }

  self.store = new(ArticleStore)
  self.store.directory = self.conf.store["base_dir"]
  self.store.Init()

  self.api = new(SRNdAPI)
  self.api.Init(self)

  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  if self.sync_on_start {
    log.Println("sync on start")
  }
  self.bind_addr = self.conf.daemon["bind"]
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  return true
}
<|endoftext|>


<commit_before>package resolves

import (
	"strconv"
	"strings"

	. "github.com/wanliu/flow/builtin/luis"
)

type ItemResolve struct {
	// Original_string string
	Resolved   bool
	Name       string
	Price      float64
	Quantity   int
	Product    string
	Resolution Resolution
}

func (r ItemResolve) Hint() string {
	// choses := "\n" + strings.Join(r.Resolution.Values, "\n")

	var result string

	if r.Product == "" && len(r.Resolution.Values) > 0 {
		index := 1
		choses := "\n"

		for _, value := range r.Resolution.Values {
			choses = choses + strconv.Itoa(index) + ": " + value + "\n"
			index = index + 1
		}

		choses = choses + "\n"

		result = "我们有下列的 " + r.Name + " 产品:" + choses + "请输入序号选择"
	} else if r.Quantity == 0 {
		result = "请告诉我您要购买的数量\n"
	}

	return result

}

func (r *ItemResolve) Solve(luis ResultParams) (bool, string, string) {
	if luis.TopScoringIntent.Intent == "选择" {
		number := strings.Trim(luis.Entities[0].Resolution.Value, " ")
		chose, _ := strconv.ParseInt(number, 10, 64)
		inNum := int(chose)

		if r.Product == "" {
			if len(r.Resolution.Values) >= inNum {
				prdName := r.Resolution.Values[chose-1]

				r.Product = prdName
				r.CheckResolved()

				return true, "已选择" + prdName, "err"
			} else {
				return false, "", "超出选择范围\n" + r.Hint()
			}
		} else if r.Quantity == 0 {
			if 0 < inNum {
				r.Quantity = inNum
				r.CheckResolved()

				return true, "购买的数量为：" + strconv.Itoa(inNum), ""
			} else {
				return false, "", "购买的数量必须大于零, 请重新输入\n"
			}
		}

		return false, "", "错误的操作，没有可供选择的商品"
	} else {
		return false, "", "无效的输入\n" + r.Hint()
	}
}

func (r *ItemResolve) CheckResolved() {
	if len(r.Resolution.Values) == 0 {
		r.Product = r.Name
	} else if len(r.Resolution.Values) == 1 {
		r.Product = r.Resolution.Values[0]
	}

	if r.Product != "" && r.Quantity > 0 {
		r.Resolved = true
	}
}
<commit_msg>invalid input hint<commit_after>package resolves

import (
	"strconv"
	"strings"

	. "github.com/wanliu/flow/builtin/luis"
)

type ItemResolve struct {
	// Original_string string
	Resolved   bool
	Name       string
	Price      float64
	Quantity   int
	Product    string
	Resolution Resolution
}

func (r ItemResolve) Hint() string {
	// choses := "\n" + strings.Join(r.Resolution.Values, "\n")

	var result string

	if r.Product == "" && len(r.Resolution.Values) > 0 {
		index := 1
		choses := "\n"

		for _, value := range r.Resolution.Values {
			choses = choses + strconv.Itoa(index) + ": " + value + "\n"
			index = index + 1
		}

		choses = choses + "\n"

		result = "我们有下列的 " + r.Name + " 产品:" + choses + "请输入序号选择"
	} else if r.Quantity == 0 {
		result = "请告诉我您要购买的数量\n"
	}

	return result

}

func (r *ItemResolve) Solve(luis ResultParams) (bool, string, string) {
	if luis.TopScoringIntent.Intent == "选择" {
		number := strings.Trim(luis.Entities[0].Resolution.Value, " ")
		chose, _ := strconv.ParseInt(number, 10, 64)
		inNum := int(chose)

		if r.Product == "" {
			if len(r.Resolution.Values) >= inNum {
				prdName := r.Resolution.Values[chose-1]

				r.Product = prdName
				r.CheckResolved()

				return true, "已选择" + prdName, "err"
			} else {
				return false, "", "超出选择范围\n" + r.Hint()
			}
		} else if r.Quantity == 0 {
			if 0 < inNum {
				r.Quantity = inNum
				r.CheckResolved()

				return true, "购买的数量为：" + strconv.Itoa(inNum), ""
			} else {
				return false, "", "购买的数量必须大于零, 请重新输入\n"
			}
		}

		return false, "", "错误的操作，没有可供选择的商品"
	} else {
		return false, "", "无效的输入: \"" + luis.Query + "\"。\n" + r.Hint()
	}
}

func (r *ItemResolve) CheckResolved() {
	if len(r.Resolution.Values) == 0 {
		r.Product = r.Name
	} else if len(r.Resolution.Values) == 1 {
		r.Product = r.Resolution.Values[0]
	}

	if r.Product != "" && r.Quantity > 0 {
		r.Resolved = true
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apimachinery

import (
	"context"
	"fmt"
	"reflect"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	watchtools "k8s.io/client-go/tools/watch"
	"k8s.io/kubernetes/test/integration/framework"
)

func noopNormalization(output []string) []string {
	return output
}

func normalizeInformerOutputFunc(initialVal string) func(output []string) []string {
	return func(output []string) []string {
		result := make([]string, 0, len(output))

		// Removes initial value and all of its direct repetitions
		lastVal := initialVal
		for _, v := range output {
			// Make values unique as informer(List+Watch) duplicates some events
			if v == lastVal {
				continue
			}
			result = append(result, v)
			lastVal = v
		}

		return result
	}
}

func noop() {}

func TestWatchRestartsIfTimeoutNotReached(t *testing.T) {
	// Has to be longer than 5 seconds
	timeout := 30 * time.Second

	// Set up an API server
	controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig()
	// Timeout is set random between MinRequestTimeout and 2x
	controlPlaneConfig.GenericConfig.MinRequestTimeout = int(timeout.Seconds()) / 4
	_, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig)
	defer closeFn()

	config := &restclient.Config{
		Host: s.URL,
	}

	namespaceObject := framework.CreateTestingNamespace("retry-watch", s, t)
	defer framework.DeleteTestingNamespace(namespaceObject, s, t)

	getListFunc := func(c *kubernetes.Clientset, secret *corev1.Secret) func(options metav1.ListOptions) *corev1.SecretList {
		return func(options metav1.ListOptions) *corev1.SecretList {
			options.FieldSelector = fields.OneTermEqualSelector("metadata.name", secret.Name).String()
			res, err := c.CoreV1().Secrets(secret.Namespace).List(context.TODO(), options)
			if err != nil {
				t.Fatalf("Failed to list Secrets: %v", err)
			}
			return res
		}
	}

	getWatchFunc := func(c *kubernetes.Clientset, secret *corev1.Secret) func(options metav1.ListOptions) (watch.Interface, error) {
		return func(options metav1.ListOptions) (watch.Interface, error) {
			options.FieldSelector = fields.OneTermEqualSelector("metadata.name", secret.Name).String()
			res, err := c.CoreV1().Secrets(secret.Namespace).Watch(context.TODO(), options)
			if err != nil {
				t.Fatalf("Failed to create a watcher on Secrets: %v", err)
			}
			return res, err
		}
	}

	generateEvents := func(t *testing.T, c *kubernetes.Clientset, secret *corev1.Secret, referenceOutput *[]string, stopChan chan struct{}, stoppedChan chan struct{}) {
		defer close(stoppedChan)
		counter := 0

		// These 5 seconds are here to protect against a race at the end when we could write something there at the same time as watch.Until ends
		softTimeout := timeout - 5*time.Second
		if softTimeout < 0 {
			panic("Timeout has to be grater than 5 seconds!")
		}
		endChannel := time.After(softTimeout)
		for {
			select {
			// TODO: get this lower once we figure out how to extend ETCD cache
			case <-time.After(1000 * time.Millisecond):
				counter = counter + 1

				patch := fmt.Sprintf(`{"metadata": {"annotations": {"count": "%d"}}}`, counter)
				_, err := c.CoreV1().Secrets(secret.Namespace).Patch(context.TODO(), secret.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{})
				if err != nil {
					t.Fatalf("Failed to patch secret: %v", err)
				}

				*referenceOutput = append(*referenceOutput, fmt.Sprintf("%d", counter))
			case <-endChannel:
				return
			case <-stopChan:
				return
			}
		}
	}

	initialCount := "0"
	newTestSecret := func(name string) *corev1.Secret {
		return &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: namespaceObject.Name,
				Annotations: map[string]string{
					"count": initialCount,
				},
			},
			Data: map[string][]byte{
				"data": []byte("value1\n"),
			},
		}
	}

	tt := []struct {
		name                string
		succeed             bool
		secret              *corev1.Secret
		getWatcher          func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func())
		normalizeOutputFunc func(referenceOutput []string) []string
	}{
		{
			name:    "regular watcher should fail",
			succeed: false,
			secret:  newTestSecret("secret-01"),
			getWatcher: func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func()) {
				options := metav1.ListOptions{
					ResourceVersion: secret.ResourceVersion,
				}
				w, err := getWatchFunc(c, secret)(options)
				return w, err, noop
			}, // regular watcher; unfortunately destined to fail
			normalizeOutputFunc: noopNormalization,
		},
		{
			name:    "RetryWatcher survives closed watches",
			succeed: true,
			secret:  newTestSecret("secret-02"),
			getWatcher: func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func()) {
				lw := &cache.ListWatch{
					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
						return getWatchFunc(c, secret)(options)
					},
				}
				w, err := watchtools.NewRetryWatcher(secret.ResourceVersion, lw)
				return w, err, func() { <-w.Done() }
			},
			normalizeOutputFunc: noopNormalization,
		},
		{
			name:    "InformerWatcher survives closed watches",
			succeed: true,
			secret:  newTestSecret("secret-03"),
			getWatcher: func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func()) {
				lw := &cache.ListWatch{
					ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
						return getListFunc(c, secret)(options), nil
					},
					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
						return getWatchFunc(c, secret)(options)
					},
				}
				_, _, w, done := watchtools.NewIndexerInformerWatcher(lw, &corev1.Secret{})
				return w, nil, func() { <-done }
			},
			normalizeOutputFunc: normalizeInformerOutputFunc(initialCount),
		},
	}

	t.Run("group", func(t *testing.T) {
		for _, tmptc := range tt {
			tc := tmptc // we need to copy it for parallel runs
			t.Run(tc.name, func(t *testing.T) {
				t.Parallel()
				c, err := kubernetes.NewForConfig(config)
				if err != nil {
					t.Fatalf("Failed to create clientset: %v", err)
				}

				secret, err := c.CoreV1().Secrets(tc.secret.Namespace).Create(context.TODO(), tc.secret, metav1.CreateOptions{})
				if err != nil {
					t.Fatalf("Failed to create testing secret %s/%s: %v", tc.secret.Namespace, tc.secret.Name, err)
				}

				watcher, err, doneFn := tc.getWatcher(c, secret)
				if err != nil {
					t.Fatalf("Failed to create watcher: %v", err)
				}
				defer doneFn()

				var referenceOutput []string
				var output []string
				stopChan := make(chan struct{})
				stoppedChan := make(chan struct{})
				go generateEvents(t, c, secret, &referenceOutput, stopChan, stoppedChan)

				// Record current time to be able to asses if the timeout has been reached
				startTime := time.Now()
				ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout)
				defer cancel()
				_, err = watchtools.UntilWithoutRetry(ctx, watcher, func(event watch.Event) (bool, error) {
					s, ok := event.Object.(*corev1.Secret)
					if !ok {
						t.Fatalf("Received an object that is not a Secret: %#v", event.Object)
					}
					output = append(output, s.Annotations["count"])
					// Watch will never end voluntarily
					return false, nil
				})
				watchDuration := time.Since(startTime)
				close(stopChan)
				<-stoppedChan

				output = tc.normalizeOutputFunc(output)

				t.Logf("Watch duration: %v; timeout: %v", watchDuration, timeout)

				if err == nil && !tc.succeed {
					t.Fatalf("Watch should have timed out but it exited without an error!")
				}

				if err != wait.ErrWaitTimeout && tc.succeed {
					t.Fatalf("Watch exited with error: %v!", err)
				}

				if watchDuration < timeout && tc.succeed {
					t.Fatalf("Watch should have timed out after %v but it timed out prematurely after %v!", timeout, watchDuration)
				}

				if watchDuration >= timeout && !tc.succeed {
					t.Fatalf("Watch should have timed out but it succeeded!")
				}

				if tc.succeed && !reflect.DeepEqual(referenceOutput, output) {
					t.Fatalf("Reference and real output differ! We must have lost some events or read some multiple times!\nRef:  %#v\nReal: %#v", referenceOutput, output)
				}
			})
		}
	})
}
<commit_msg>fix govet on golang 1.18<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apimachinery

import (
	"context"
	"fmt"
	"reflect"
	"testing"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	watchtools "k8s.io/client-go/tools/watch"
	"k8s.io/kubernetes/test/integration/framework"
)

func noopNormalization(output []string) []string {
	return output
}

func normalizeInformerOutputFunc(initialVal string) func(output []string) []string {
	return func(output []string) []string {
		result := make([]string, 0, len(output))

		// Removes initial value and all of its direct repetitions
		lastVal := initialVal
		for _, v := range output {
			// Make values unique as informer(List+Watch) duplicates some events
			if v == lastVal {
				continue
			}
			result = append(result, v)
			lastVal = v
		}

		return result
	}
}

func noop() {}

func TestWatchRestartsIfTimeoutNotReached(t *testing.T) {
	// Has to be longer than 5 seconds
	timeout := 30 * time.Second

	// Set up an API server
	controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig()
	// Timeout is set random between MinRequestTimeout and 2x
	controlPlaneConfig.GenericConfig.MinRequestTimeout = int(timeout.Seconds()) / 4
	_, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig)
	defer closeFn()

	config := &restclient.Config{
		Host: s.URL,
	}

	namespaceObject := framework.CreateTestingNamespace("retry-watch", s, t)
	defer framework.DeleteTestingNamespace(namespaceObject, s, t)

	getListFunc := func(c *kubernetes.Clientset, secret *corev1.Secret) func(options metav1.ListOptions) *corev1.SecretList {
		return func(options metav1.ListOptions) *corev1.SecretList {
			options.FieldSelector = fields.OneTermEqualSelector("metadata.name", secret.Name).String()
			res, err := c.CoreV1().Secrets(secret.Namespace).List(context.TODO(), options)
			if err != nil {
				t.Fatalf("Failed to list Secrets: %v", err)
			}
			return res
		}
	}

	getWatchFunc := func(c *kubernetes.Clientset, secret *corev1.Secret) func(options metav1.ListOptions) (watch.Interface, error) {
		return func(options metav1.ListOptions) (watch.Interface, error) {
			options.FieldSelector = fields.OneTermEqualSelector("metadata.name", secret.Name).String()
			res, err := c.CoreV1().Secrets(secret.Namespace).Watch(context.TODO(), options)
			if err != nil {
				t.Fatalf("Failed to create a watcher on Secrets: %v", err)
			}
			return res, err
		}
	}

	generateEvents := func(t *testing.T, c *kubernetes.Clientset, secret *corev1.Secret, referenceOutput *[]string, stopChan chan struct{}, stoppedChan chan struct{}) {
		defer close(stoppedChan)
		counter := 0

		// These 5 seconds are here to protect against a race at the end when we could write something there at the same time as watch.Until ends
		softTimeout := timeout - 5*time.Second
		if softTimeout < 0 {
			panic("Timeout has to be grater than 5 seconds!")
		}
		endChannel := time.After(softTimeout)
		for {
			select {
			// TODO: get this lower once we figure out how to extend ETCD cache
			case <-time.After(1000 * time.Millisecond):
				counter = counter + 1

				patch := fmt.Sprintf(`{"metadata": {"annotations": {"count": "%d"}}}`, counter)
				_, err := c.CoreV1().Secrets(secret.Namespace).Patch(context.TODO(), secret.Name, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{})
				if err != nil {
					panic(fmt.Sprintf("Failed to patch secret: %v", err))
				}

				*referenceOutput = append(*referenceOutput, fmt.Sprintf("%d", counter))
			case <-endChannel:
				return
			case <-stopChan:
				return
			}
		}
	}

	initialCount := "0"
	newTestSecret := func(name string) *corev1.Secret {
		return &corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: namespaceObject.Name,
				Annotations: map[string]string{
					"count": initialCount,
				},
			},
			Data: map[string][]byte{
				"data": []byte("value1\n"),
			},
		}
	}

	tt := []struct {
		name                string
		succeed             bool
		secret              *corev1.Secret
		getWatcher          func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func())
		normalizeOutputFunc func(referenceOutput []string) []string
	}{
		{
			name:    "regular watcher should fail",
			succeed: false,
			secret:  newTestSecret("secret-01"),
			getWatcher: func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func()) {
				options := metav1.ListOptions{
					ResourceVersion: secret.ResourceVersion,
				}
				w, err := getWatchFunc(c, secret)(options)
				return w, err, noop
			}, // regular watcher; unfortunately destined to fail
			normalizeOutputFunc: noopNormalization,
		},
		{
			name:    "RetryWatcher survives closed watches",
			succeed: true,
			secret:  newTestSecret("secret-02"),
			getWatcher: func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func()) {
				lw := &cache.ListWatch{
					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
						return getWatchFunc(c, secret)(options)
					},
				}
				w, err := watchtools.NewRetryWatcher(secret.ResourceVersion, lw)
				return w, err, func() { <-w.Done() }
			},
			normalizeOutputFunc: noopNormalization,
		},
		{
			name:    "InformerWatcher survives closed watches",
			succeed: true,
			secret:  newTestSecret("secret-03"),
			getWatcher: func(c *kubernetes.Clientset, secret *corev1.Secret) (watch.Interface, error, func()) {
				lw := &cache.ListWatch{
					ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
						return getListFunc(c, secret)(options), nil
					},
					WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
						return getWatchFunc(c, secret)(options)
					},
				}
				_, _, w, done := watchtools.NewIndexerInformerWatcher(lw, &corev1.Secret{})
				return w, nil, func() { <-done }
			},
			normalizeOutputFunc: normalizeInformerOutputFunc(initialCount),
		},
	}

	t.Run("group", func(t *testing.T) {
		for _, tmptc := range tt {
			tc := tmptc // we need to copy it for parallel runs
			t.Run(tc.name, func(t *testing.T) {
				t.Parallel()
				c, err := kubernetes.NewForConfig(config)
				if err != nil {
					t.Fatalf("Failed to create clientset: %v", err)
				}

				secret, err := c.CoreV1().Secrets(tc.secret.Namespace).Create(context.TODO(), tc.secret, metav1.CreateOptions{})
				if err != nil {
					t.Fatalf("Failed to create testing secret %s/%s: %v", tc.secret.Namespace, tc.secret.Name, err)
				}

				watcher, err, doneFn := tc.getWatcher(c, secret)
				if err != nil {
					t.Fatalf("Failed to create watcher: %v", err)
				}
				defer doneFn()

				var referenceOutput []string
				var output []string
				stopChan := make(chan struct{})
				stoppedChan := make(chan struct{})
				go generateEvents(t, c, secret, &referenceOutput, stopChan, stoppedChan)

				// Record current time to be able to asses if the timeout has been reached
				startTime := time.Now()
				ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout)
				defer cancel()
				_, err = watchtools.UntilWithoutRetry(ctx, watcher, func(event watch.Event) (bool, error) {
					s, ok := event.Object.(*corev1.Secret)
					if !ok {
						t.Fatalf("Received an object that is not a Secret: %#v", event.Object)
					}
					output = append(output, s.Annotations["count"])
					// Watch will never end voluntarily
					return false, nil
				})
				watchDuration := time.Since(startTime)
				close(stopChan)
				<-stoppedChan

				output = tc.normalizeOutputFunc(output)

				t.Logf("Watch duration: %v; timeout: %v", watchDuration, timeout)

				if err == nil && !tc.succeed {
					t.Fatalf("Watch should have timed out but it exited without an error!")
				}

				if err != wait.ErrWaitTimeout && tc.succeed {
					t.Fatalf("Watch exited with error: %v!", err)
				}

				if watchDuration < timeout && tc.succeed {
					t.Fatalf("Watch should have timed out after %v but it timed out prematurely after %v!", timeout, watchDuration)
				}

				if watchDuration >= timeout && !tc.succeed {
					t.Fatalf("Watch should have timed out but it succeeded!")
				}

				if tc.succeed && !reflect.DeepEqual(referenceOutput, output) {
					t.Fatalf("Reference and real output differ! We must have lost some events or read some multiple times!\nRef:  %#v\nReal: %#v", referenceOutput, output)
				}
			})
		}
	})
}
<|endoftext|>


<commit_before>// Package web is a lightweight web framework for Go. It's ideal for
// writing simple, performant backend web services.
package web

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"github.com/codegangsta/martini"
	"io/ioutil"
	"mime"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// A Context object is created for every incoming HTTP request, and is
// passed to handlers as an optional first argument. It provides information
// about the request, including the http.Request object, the GET and POST params,
// and acts as a Writer for the response.
type Context struct {
	Request      *http.Request
	Params       map[string]string
	cookieSecret string
	http.ResponseWriter
}

func ContextWithCookieSecret(secret string) martini.Handler {
	return func(w http.ResponseWriter, req *http.Request, mc martini.Context) {
		ctx := &Context{req, map[string]string{}, secret, w}

		//set some default headers
		tm := time.Now().UTC()

		//ignore errors from ParseForm because it's usually harmless.
		req.ParseForm()
		if len(req.Form) > 0 {
			for k, v := range req.Form {
				ctx.Params[k] = v[0]
			}
		}
		ctx.SetHeader("Date", webTime(tm), true)
		//Set the default content-type
		ctx.SetHeader("Content-Type", "text/html; charset=utf-8", true)
		// set martini context for web.Context
		mc.Map(ctx)
	}
}

// internal utility methods
func webTime(t time.Time) string {
	ftime := t.Format(time.RFC1123)
	if strings.HasSuffix(ftime, "UTC") {
		ftime = ftime[0:len(ftime)-3] + "GMT"
	}
	return ftime
}

// WriteString writes string data into the response object.
func (ctx *Context) WriteString(content string) {
	ctx.ResponseWriter.Write([]byte(content))
}

// Abort is a helper method that sends an HTTP header and an optional
// body. It is useful for returning 4xx or 5xx errors.
// Once it has been called, any return value from the handler will
// not be written to the response.
func (ctx *Context) Abort(status int, body string) {
	ctx.ResponseWriter.WriteHeader(status)
	ctx.ResponseWriter.Write([]byte(body))
}

// Redirect is a helper method for 3xx redirects.
func (ctx *Context) Redirect(status int, url_ string) {
	ctx.ResponseWriter.Header().Set("Location", url_)
	ctx.ResponseWriter.WriteHeader(status)
	ctx.ResponseWriter.Write([]byte("Redirecting to: " + url_))
}

// Notmodified writes a 304 HTTP response
func (ctx *Context) NotModified() {
	ctx.ResponseWriter.WriteHeader(304)
}

// NotFound writes a 404 HTTP response
func (ctx *Context) NotFound(message string) {
	ctx.ResponseWriter.WriteHeader(404)
	ctx.ResponseWriter.Write([]byte(message))
}

//Unauthorized writes a 401 HTTP response
func (ctx *Context) Unauthorized() {
	ctx.ResponseWriter.WriteHeader(401)
}

//Forbidden writes a 403 HTTP response
func (ctx *Context) Forbidden() {
	ctx.ResponseWriter.WriteHeader(403)
}

// ContentType sets the Content-Type header for an HTTP response.
// For example, ctx.ContentType("json") sets the content-type to "application/json"
// If the supplied value contains a slash (/) it is set as the Content-Type
// verbatim. The return value is the content type as it was
// set, or an empty string if none was found.
func (ctx *Context) ContentType(val string) string {
	var ctype string
	if strings.ContainsRune(val, '/') {
		ctype = val
	} else {
		if !strings.HasPrefix(val, ".") {
			val = "." + val
		}
		ctype = mime.TypeByExtension(val)
	}
	if ctype != "" {
		ctx.Header().Set("Content-Type", ctype)
	}
	return ctype
}

// SetHeader sets a response header. If `unique` is true, the current value
// of that header will be overwritten . If false, it will be appended.
func (ctx *Context) SetHeader(hdr string, val string, unique bool) {
	if unique {
		ctx.Header().Set(hdr, val)
	} else {
		ctx.Header().Add(hdr, val)
	}
}

// SetCookie adds a cookie header to the response.
func (ctx *Context) SetCookie(cookie *http.Cookie) {
	ctx.SetHeader("Set-Cookie", cookie.String(), false)
}

func getCookieSig(key string, val []byte, timestamp string) string {
	hm := hmac.New(sha1.New, []byte(key))

	hm.Write(val)
	hm.Write([]byte(timestamp))

	hex := fmt.Sprintf("%02x", hm.Sum(nil))
	return hex
}

// NewCookie is a helper method that returns a new http.Cookie object.
// Duration is specified in seconds. If the duration is zero, the cookie is permanent.
// This can be used in conjunction with ctx.SetCookie.
func NewCookie(name string, value string, age int64) *http.Cookie {
	var utctime time.Time
	if age == 0 {
		// 2^31 - 1 seconds (roughly 2038)
		utctime = time.Unix(2147483647, 0)
	} else {
		utctime = time.Unix(time.Now().Unix()+age, 0)
	}
	return &http.Cookie{Name: name, Value: value, Expires: utctime}
}

func (ctx *Context) SetSecureCookie(name string, val string, age int64) {
	//base64 encode the val
	if len(ctx.cookieSecret) == 0 {
		return
	}
	var buf bytes.Buffer
	encoder := base64.NewEncoder(base64.StdEncoding, &buf)
	encoder.Write([]byte(val))
	encoder.Close()
	vs := buf.String()
	vb := buf.Bytes()
	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
	sig := getCookieSig(ctx.cookieSecret, vb, timestamp)
	cookie := strings.Join([]string{vs, timestamp, sig}, "|")
	ctx.SetCookie(NewCookie(name, cookie, age))
}

func (ctx *Context) GetSecureCookie(name string) (string, bool) {
	for _, cookie := range ctx.Request.Cookies() {
		if cookie.Name != name {
			continue
		}

		parts := strings.SplitN(cookie.Value, "|", 3)

		val := parts[0]
		timestamp := parts[1]
		sig := parts[2]

		if getCookieSig(ctx.cookieSecret, []byte(val), timestamp) != sig {
			return "", false
		}

		ts, _ := strconv.ParseInt(timestamp, 0, 64)

		if time.Now().Unix()-31*86400 > ts {
			return "", false
		}

		buf := bytes.NewBufferString(val)
		encoder := base64.NewDecoder(base64.StdEncoding, buf)

		res, _ := ioutil.ReadAll(encoder)
		return string(res), true
	}
	return "", false
}
<commit_msg>update comments of ContextWithCookieSecret<commit_after>// Package web is a lightweight web framework for Go. It's ideal for
// writing simple, performant backend web services.
package web

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"github.com/codegangsta/martini"
	"io/ioutil"
	"mime"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// A Context object is created for every incoming HTTP request, and is
// passed to handlers as an optional first argument. It provides information
// about the request, including the http.Request object, the GET and POST params,
// and acts as a Writer for the response.
type Context struct {
	Request      *http.Request
	Params       map[string]string
	cookieSecret string
	http.ResponseWriter
}

// if cookie secret is set to "", then SetSecureCookie would not work
func ContextWithCookieSecret(secret string) martini.Handler {
	return func(w http.ResponseWriter, req *http.Request, mc martini.Context) {
		ctx := &Context{req, map[string]string{}, secret, w}
		//set some default headers
		tm := time.Now().UTC()

		//ignore errors from ParseForm because it's usually harmless.
		req.ParseForm()
		if len(req.Form) > 0 {
			for k, v := range req.Form {
				ctx.Params[k] = v[0]
			}
		}
		ctx.SetHeader("Date", webTime(tm), true)
		//Set the default content-type
		ctx.SetHeader("Content-Type", "text/html; charset=utf-8", true)
		// set martini context for web.Context
		mc.Map(ctx)
	}
}

// internal utility methods
func webTime(t time.Time) string {
	ftime := t.Format(time.RFC1123)
	if strings.HasSuffix(ftime, "UTC") {
		ftime = ftime[0:len(ftime)-3] + "GMT"
	}
	return ftime
}

// WriteString writes string data into the response object.
func (ctx *Context) WriteString(content string) {
	ctx.ResponseWriter.Write([]byte(content))
}

// Abort is a helper method that sends an HTTP header and an optional
// body. It is useful for returning 4xx or 5xx errors.
// Once it has been called, any return value from the handler will
// not be written to the response.
func (ctx *Context) Abort(status int, body string) {
	ctx.ResponseWriter.WriteHeader(status)
	ctx.ResponseWriter.Write([]byte(body))
}

// Redirect is a helper method for 3xx redirects.
func (ctx *Context) Redirect(status int, url_ string) {
	ctx.ResponseWriter.Header().Set("Location", url_)
	ctx.ResponseWriter.WriteHeader(status)
	ctx.ResponseWriter.Write([]byte("Redirecting to: " + url_))
}

// Notmodified writes a 304 HTTP response
func (ctx *Context) NotModified() {
	ctx.ResponseWriter.WriteHeader(304)
}

// NotFound writes a 404 HTTP response
func (ctx *Context) NotFound(message string) {
	ctx.ResponseWriter.WriteHeader(404)
	ctx.ResponseWriter.Write([]byte(message))
}

//Unauthorized writes a 401 HTTP response
func (ctx *Context) Unauthorized() {
	ctx.ResponseWriter.WriteHeader(401)
}

//Forbidden writes a 403 HTTP response
func (ctx *Context) Forbidden() {
	ctx.ResponseWriter.WriteHeader(403)
}

// ContentType sets the Content-Type header for an HTTP response.
// For example, ctx.ContentType("json") sets the content-type to "application/json"
// If the supplied value contains a slash (/) it is set as the Content-Type
// verbatim. The return value is the content type as it was
// set, or an empty string if none was found.
func (ctx *Context) ContentType(val string) string {
	var ctype string
	if strings.ContainsRune(val, '/') {
		ctype = val
	} else {
		if !strings.HasPrefix(val, ".") {
			val = "." + val
		}
		ctype = mime.TypeByExtension(val)
	}
	if ctype != "" {
		ctx.Header().Set("Content-Type", ctype)
	}
	return ctype
}

// SetHeader sets a response header. If `unique` is true, the current value
// of that header will be overwritten . If false, it will be appended.
func (ctx *Context) SetHeader(hdr string, val string, unique bool) {
	if unique {
		ctx.Header().Set(hdr, val)
	} else {
		ctx.Header().Add(hdr, val)
	}
}

// SetCookie adds a cookie header to the response.
func (ctx *Context) SetCookie(cookie *http.Cookie) {
	ctx.SetHeader("Set-Cookie", cookie.String(), false)
}

func getCookieSig(key string, val []byte, timestamp string) string {
	hm := hmac.New(sha1.New, []byte(key))

	hm.Write(val)
	hm.Write([]byte(timestamp))

	hex := fmt.Sprintf("%02x", hm.Sum(nil))
	return hex
}

// NewCookie is a helper method that returns a new http.Cookie object.
// Duration is specified in seconds. If the duration is zero, the cookie is permanent.
// This can be used in conjunction with ctx.SetCookie.
func NewCookie(name string, value string, age int64) *http.Cookie {
	var utctime time.Time
	if age == 0 {
		// 2^31 - 1 seconds (roughly 2038)
		utctime = time.Unix(2147483647, 0)
	} else {
		utctime = time.Unix(time.Now().Unix()+age, 0)
	}
	return &http.Cookie{Name: name, Value: value, Expires: utctime}
}

func (ctx *Context) SetSecureCookie(name string, val string, age int64) {
	//base64 encode the val
	if len(ctx.cookieSecret) == 0 {
		return
	}
	var buf bytes.Buffer
	encoder := base64.NewEncoder(base64.StdEncoding, &buf)
	encoder.Write([]byte(val))
	encoder.Close()
	vs := buf.String()
	vb := buf.Bytes()
	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
	sig := getCookieSig(ctx.cookieSecret, vb, timestamp)
	cookie := strings.Join([]string{vs, timestamp, sig}, "|")
	ctx.SetCookie(NewCookie(name, cookie, age))
}

func (ctx *Context) GetSecureCookie(name string) (string, bool) {
	for _, cookie := range ctx.Request.Cookies() {
		if cookie.Name != name {
			continue
		}

		parts := strings.SplitN(cookie.Value, "|", 3)

		val := parts[0]
		timestamp := parts[1]
		sig := parts[2]

		if getCookieSig(ctx.cookieSecret, []byte(val), timestamp) != sig {
			return "", false
		}

		ts, _ := strconv.ParseInt(timestamp, 0, 64)

		if time.Now().Unix()-31*86400 > ts {
			return "", false
		}

		buf := bytes.NewBufferString(val)
		encoder := base64.NewDecoder(base64.StdEncoding, buf)

		res, _ := ioutil.ReadAll(encoder)
		return string(res), true
	}
	return "", false
}
<|endoftext|>