<commit_before>// package swarm implements a connection muxer with a pair of channels
// to synchronize all network communication.
package swarm

import (
	"fmt"
	"sync"
	"time"

	metrics "github.com/ipfs/go-libp2p/p2p/metrics"
	mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn"
	inet "github.com/ipfs/go-libp2p/p2p/net"
	conn "github.com/ipfs/go-libp2p/p2p/net/conn"
	filter "github.com/ipfs/go-libp2p/p2p/net/filter"
	addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr"
	transport "github.com/ipfs/go-libp2p/p2p/net/transport"
	peer "github.com/ipfs/go-libp2p/p2p/peer"

	ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream"
	"gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess"
	goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context"
	ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr"
	pst "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer"
	psmss "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer/multistream"
	mafilter "gx/ipfs/QmW3Q7RQa8D1qCEEeyKCBV1drgFxvHBqAZ3zgCujEwKpHD/multiaddr-filter"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"

	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
)

var log = logging.Logger("swarm2")

var PSTransport pst.Transport

func init() {
	PSTransport = psmss.NewTransport()
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
//
// Uses peerstream.Swarm
type Swarm struct {
	swarm *ps.Swarm
	local peer.ID
	peers peer.Peerstore
	connh ConnHandler

	dsync dialsync
	backf dialbackoff
	dialT time.Duration // mainly for tests

	dialer *conn.Dialer

	notifmu sync.RWMutex
	notifs  map[inet.Notifiee]ps.Notifiee

	transports []transport.Transport

	// filters for addresses that shouldnt be dialed
	Filters *filter.Filters

	// file descriptor rate limited
	fdRateLimit chan struct{}

	proc goprocess.Process
	ctx  context.Context
	bwc  metrics.Reporter
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr,
	local peer.ID, peers peer.Peerstore, bwc metrics.Reporter) (*Swarm, error) {

	listenAddrs, err := filterAddrs(listenAddrs)
	if err != nil {
		return nil, err
	}

	wrap := func(c transport.Conn) transport.Conn {
		return mconn.WrapConn(bwc, c)
	}

	s := &Swarm{
		swarm:       ps.NewSwarm(PSTransport),
		local:       local,
		peers:       peers,
		ctx:         ctx,
		dialT:       DialTimeout,
		notifs:      make(map[inet.Notifiee]ps.Notifiee),
		transports:  []transport.Transport{transport.NewTCPTransport()},
		bwc:         bwc,
		fdRateLimit: make(chan struct{}, concurrentFdDials),
		Filters:     filter.NewFilters(),
		dialer:      conn.NewDialer(local, peers.PrivKey(local), wrap),
	}

	// configure Swarm
	s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown)
	s.SetConnHandler(nil) // make sure to setup our own conn handler.

	err = s.setupInterfaces(listenAddrs)
	if err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Swarm) teardown() error {
	return s.swarm.Close()
}

func (s *Swarm) AddAddrFilter(f string) error {
	m, err := mafilter.NewMask(f)
	if err != nil {
		return err
	}

	s.Filters.AddDialFilter(m)
	return nil
}
func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) {
	if len(listenAddrs) > 0 {
		filtered := addrutil.FilterUsableAddrs(listenAddrs)
		if len(filtered) < 1 {
			return nil, fmt.Errorf("swarm cannot use any addr in: %s", listenAddrs)
		}
		listenAddrs = filtered
	}
	return listenAddrs, nil
}

func (s *Swarm) Listen(addrs ...ma.Multiaddr) error {
	addrs, err := filterAddrs(addrs)
	if err != nil {
		return err
	}

	return s.setupInterfaces(addrs)
}

// Process returns the Process of the swarm
func (s *Swarm) Process() goprocess.Process {
	return s.proc
}

// Context returns the context of the swarm
func (s *Swarm) Context() context.Context {
	return s.ctx
}

// Close stops the Swarm.
func (s *Swarm) Close() error {
	return s.proc.Close()
}

// StreamSwarm returns the underlying peerstream.Swarm
func (s *Swarm) StreamSwarm() *ps.Swarm {
	return s.swarm
}

// SetConnHandler assigns the handler for new connections.
// See peerstream. You will rarely use this. See SetStreamHandler
func (s *Swarm) SetConnHandler(handler ConnHandler) {

	// handler is nil if user wants to clear the old handler.
	if handler == nil {
		s.swarm.SetConnHandler(func(psconn *ps.Conn) {
			s.connHandler(psconn)
		})
		return
	}

	s.swarm.SetConnHandler(func(psconn *ps.Conn) {
		// sc is nil if closed in our handler.
		if sc := s.connHandler(psconn); sc != nil {
			// call the user's handler. in a goroutine for sync safety.
			go handler(sc)
		}
	})
}

// SetStreamHandler assigns the handler for new streams.
// See peerstream.
func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) {
	s.swarm.SetStreamHandler(func(s *ps.Stream) {
		handler(wrapStream(s))
	})
}

// NewStreamWithPeer creates a new stream on any available connection to p
func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, error) {
	// if we have no connections, try connecting.
	if len(s.ConnectionsToPeer(p)) == 0 {
		log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...")
		if _, err := s.Dial(ctx, p); err != nil {
			return nil, err
		}
	}
	log.Debug("Swarm: NewStreamWithPeer...")

	// TODO: think about passing a context down to NewStreamWithGroup
	st, err := s.swarm.NewStreamWithGroup(p)
	return wrapStream(st), err
}

// StreamsWithPeer returns all the live Streams to p
func (s *Swarm) StreamsWithPeer(p peer.ID) []*Stream {
	return wrapStreams(ps.StreamsWithGroup(p, s.swarm.Streams()))
}

// ConnectionsToPeer returns all the live connections to p
func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn {
	return wrapConns(ps.ConnsWithGroup(p, s.swarm.Conns()))
}

// Connections returns a slice of all connections.
func (s *Swarm) Connections() []*Conn {
	return wrapConns(s.swarm.Conns())
}

// CloseConnection removes a given peer from swarm + closes the connection
func (s *Swarm) CloseConnection(p peer.ID) error {
	conns := s.swarm.ConnsWithGroup(p) // boom.
	for _, c := range conns {
		c.Close()
	}
	return nil
}

// Peers returns a copy of the set of peers swarm is connected to.
func (s *Swarm) Peers() []peer.ID {
	conns := s.Connections()

	seen := make(map[peer.ID]struct{})
	peers := make([]peer.ID, 0, len(conns))
	for _, c := range conns {
		p := c.RemotePeer()
		if _, found := seen[p]; found {
			continue
		}

		seen[p] = struct{}{}
		peers = append(peers, p)
	}
	return peers
}

// LocalPeer returns the local peer swarm is associated to.
func (s *Swarm) LocalPeer() peer.ID {
	return s.local
}

// notifyAll sends a signal to all Notifiees
func (s *Swarm) notifyAll(notify func(inet.Notifiee)) {
	s.notifmu.RLock()
	for f := range s.notifs {
		go notify(f)
	}
	s.notifmu.RUnlock()
}

// Notify signs up Notifiee to receive signals when events happen
func (s *Swarm) Notify(f inet.Notifiee) {
	// wrap with our notifiee, to translate function calls
	n := &ps2netNotifee{net: (*Network)(s), not: f}

	s.notifmu.Lock()
	s.notifs[f] = n
	s.notifmu.Unlock()

	// register for notifications in the peer swarm.
	s.swarm.Notify(n)
}

// StopNotify unregisters Notifiee fromr receiving signals
func (s *Swarm) StopNotify(f inet.Notifiee) {
	s.notifmu.Lock()
	n, found := s.notifs[f]
	if found {
		delete(s.notifs, f)
	}
	s.notifmu.Unlock()

	if found {
		s.swarm.StopNotify(n)
	}
}

type ps2netNotifee struct {
	net *Network
	not inet.Notifiee
}

func (n *ps2netNotifee) Connected(c *ps.Conn) {
	n.not.Connected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) Disconnected(c *ps.Conn) {
	n.not.Disconnected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) OpenedStream(s *ps.Stream) {
	n.not.OpenedStream(n.net, inet.Stream((*Stream)(s)))
}

func (n *ps2netNotifee) ClosedStream(s *ps.Stream) {
	n.not.ClosedStream(n.net, inet.Stream((*Stream)(s)))
}
<commit_msg>add utp to have feature parity with go-ipfs<commit_after>// package swarm implements a connection muxer with a pair of channels
// to synchronize all network communication.
package swarm

import (
	"fmt"
	"sync"
	"time"

	metrics "github.com/ipfs/go-libp2p/p2p/metrics"
	mconn "github.com/ipfs/go-libp2p/p2p/metrics/conn"
	inet "github.com/ipfs/go-libp2p/p2p/net"
	conn "github.com/ipfs/go-libp2p/p2p/net/conn"
	filter "github.com/ipfs/go-libp2p/p2p/net/filter"
	addrutil "github.com/ipfs/go-libp2p/p2p/net/swarm/addr"
	transport "github.com/ipfs/go-libp2p/p2p/net/transport"
	peer "github.com/ipfs/go-libp2p/p2p/peer"

	ps "gx/ipfs/QmQDPXRFzRcCGPbPViQCKjzbQBkZGpLV1f9KwXnksSNcTK/go-peerstream"
	"gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess"
	goprocessctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context"
	ma "gx/ipfs/QmR3JkmZBKYXgNMNsNZawm914455Qof3PEopwuVSeXG7aV/go-multiaddr"
	pst "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer"
	psmss "gx/ipfs/QmTYr6RrJs8b63LTVwahmtytnuqzsLfNPBJp6EvmFWHbGh/go-stream-muxer/multistream"
	mafilter "gx/ipfs/QmW3Q7RQa8D1qCEEeyKCBV1drgFxvHBqAZ3zgCujEwKpHD/multiaddr-filter"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"

	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
)

var log = logging.Logger("swarm2")

var PSTransport pst.Transport

func init() {
	PSTransport = psmss.NewTransport()
}

// Swarm is a connection muxer, allowing connections to other peers to
// be opened and closed, while still using the same Chan for all
// communication. The Chan sends/receives Messages, which note the
// destination or source Peer.
//
// Uses peerstream.Swarm
type Swarm struct {
	swarm *ps.Swarm
	local peer.ID
	peers peer.Peerstore
	connh ConnHandler

	dsync dialsync
	backf dialbackoff
	dialT time.Duration // mainly for tests

	dialer *conn.Dialer

	notifmu sync.RWMutex
	notifs  map[inet.Notifiee]ps.Notifiee

	transports []transport.Transport

	// filters for addresses that shouldnt be dialed
	Filters *filter.Filters

	// file descriptor rate limited
	fdRateLimit chan struct{}

	proc goprocess.Process
	ctx  context.Context
	bwc  metrics.Reporter
}

// NewSwarm constructs a Swarm, with a Chan.
func NewSwarm(ctx context.Context, listenAddrs []ma.Multiaddr,
	local peer.ID, peers peer.Peerstore, bwc metrics.Reporter) (*Swarm, error) {

	listenAddrs, err := filterAddrs(listenAddrs)
	if err != nil {
		return nil, err
	}

	wrap := func(c transport.Conn) transport.Conn {
		return mconn.WrapConn(bwc, c)
	}

	s := &Swarm{
		swarm:  ps.NewSwarm(PSTransport),
		local:  local,
		peers:  peers,
		ctx:    ctx,
		dialT:  DialTimeout,
		notifs: make(map[inet.Notifiee]ps.Notifiee),
		transports: []transport.Transport{
			transport.NewTCPTransport(),
			transport.NewUtpTransport(),
		},
		bwc:         bwc,
		fdRateLimit: make(chan struct{}, concurrentFdDials),
		Filters:     filter.NewFilters(),
		dialer:      conn.NewDialer(local, peers.PrivKey(local), wrap),
	}

	// configure Swarm
	s.proc = goprocessctx.WithContextAndTeardown(ctx, s.teardown)
	s.SetConnHandler(nil) // make sure to setup our own conn handler.

	err = s.setupInterfaces(listenAddrs)
	if err != nil {
		return nil, err
	}

	return s, nil
}

func (s *Swarm) teardown() error {
	return s.swarm.Close()
}

func (s *Swarm) AddAddrFilter(f string) error {
	m, err := mafilter.NewMask(f)
	if err != nil {
		return err
	}

	s.Filters.AddDialFilter(m)
	return nil
}
func filterAddrs(listenAddrs []ma.Multiaddr) ([]ma.Multiaddr, error) {
	if len(listenAddrs) > 0 {
		filtered := addrutil.FilterUsableAddrs(listenAddrs)
		if len(filtered) < 1 {
			return nil, fmt.Errorf("swarm cannot use any addr in: %s", listenAddrs)
		}
		listenAddrs = filtered
	}
	return listenAddrs, nil
}

func (s *Swarm) Listen(addrs ...ma.Multiaddr) error {
	addrs, err := filterAddrs(addrs)
	if err != nil {
		return err
	}

	return s.setupInterfaces(addrs)
}

// Process returns the Process of the swarm
func (s *Swarm) Process() goprocess.Process {
	return s.proc
}

// Context returns the context of the swarm
func (s *Swarm) Context() context.Context {
	return s.ctx
}

// Close stops the Swarm.
func (s *Swarm) Close() error {
	return s.proc.Close()
}

// StreamSwarm returns the underlying peerstream.Swarm
func (s *Swarm) StreamSwarm() *ps.Swarm {
	return s.swarm
}

// SetConnHandler assigns the handler for new connections.
// See peerstream. You will rarely use this. See SetStreamHandler
func (s *Swarm) SetConnHandler(handler ConnHandler) {

	// handler is nil if user wants to clear the old handler.
	if handler == nil {
		s.swarm.SetConnHandler(func(psconn *ps.Conn) {
			s.connHandler(psconn)
		})
		return
	}

	s.swarm.SetConnHandler(func(psconn *ps.Conn) {
		// sc is nil if closed in our handler.
		if sc := s.connHandler(psconn); sc != nil {
			// call the user's handler. in a goroutine for sync safety.
			go handler(sc)
		}
	})
}

// SetStreamHandler assigns the handler for new streams.
// See peerstream.
func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) {
	s.swarm.SetStreamHandler(func(s *ps.Stream) {
		handler(wrapStream(s))
	})
}

// NewStreamWithPeer creates a new stream on any available connection to p
func (s *Swarm) NewStreamWithPeer(ctx context.Context, p peer.ID) (*Stream, error) {
	// if we have no connections, try connecting.
	if len(s.ConnectionsToPeer(p)) == 0 {
		log.Debug("Swarm: NewStreamWithPeer no connections. Attempting to connect...")
		if _, err := s.Dial(ctx, p); err != nil {
			return nil, err
		}
	}
	log.Debug("Swarm: NewStreamWithPeer...")

	// TODO: think about passing a context down to NewStreamWithGroup
	st, err := s.swarm.NewStreamWithGroup(p)
	return wrapStream(st), err
}

// StreamsWithPeer returns all the live Streams to p
func (s *Swarm) StreamsWithPeer(p peer.ID) []*Stream {
	return wrapStreams(ps.StreamsWithGroup(p, s.swarm.Streams()))
}

// ConnectionsToPeer returns all the live connections to p
func (s *Swarm) ConnectionsToPeer(p peer.ID) []*Conn {
	return wrapConns(ps.ConnsWithGroup(p, s.swarm.Conns()))
}

// Connections returns a slice of all connections.
func (s *Swarm) Connections() []*Conn {
	return wrapConns(s.swarm.Conns())
}

// CloseConnection removes a given peer from swarm + closes the connection
func (s *Swarm) CloseConnection(p peer.ID) error {
	conns := s.swarm.ConnsWithGroup(p) // boom.
	for _, c := range conns {
		c.Close()
	}
	return nil
}

// Peers returns a copy of the set of peers swarm is connected to.
func (s *Swarm) Peers() []peer.ID {
	conns := s.Connections()

	seen := make(map[peer.ID]struct{})
	peers := make([]peer.ID, 0, len(conns))
	for _, c := range conns {
		p := c.RemotePeer()
		if _, found := seen[p]; found {
			continue
		}

		seen[p] = struct{}{}
		peers = append(peers, p)
	}
	return peers
}

// LocalPeer returns the local peer swarm is associated to.
func (s *Swarm) LocalPeer() peer.ID {
	return s.local
}

// notifyAll sends a signal to all Notifiees
func (s *Swarm) notifyAll(notify func(inet.Notifiee)) {
	s.notifmu.RLock()
	for f := range s.notifs {
		go notify(f)
	}
	s.notifmu.RUnlock()
}

// Notify signs up Notifiee to receive signals when events happen
func (s *Swarm) Notify(f inet.Notifiee) {
	// wrap with our notifiee, to translate function calls
	n := &ps2netNotifee{net: (*Network)(s), not: f}

	s.notifmu.Lock()
	s.notifs[f] = n
	s.notifmu.Unlock()

	// register for notifications in the peer swarm.
	s.swarm.Notify(n)
}

// StopNotify unregisters Notifiee fromr receiving signals
func (s *Swarm) StopNotify(f inet.Notifiee) {
	s.notifmu.Lock()
	n, found := s.notifs[f]
	if found {
		delete(s.notifs, f)
	}
	s.notifmu.Unlock()

	if found {
		s.swarm.StopNotify(n)
	}
}

type ps2netNotifee struct {
	net *Network
	not inet.Notifiee
}

func (n *ps2netNotifee) Connected(c *ps.Conn) {
	n.not.Connected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) Disconnected(c *ps.Conn) {
	n.not.Disconnected(n.net, inet.Conn((*Conn)(c)))
}

func (n *ps2netNotifee) OpenedStream(s *ps.Stream) {
	n.not.OpenedStream(n.net, inet.Stream((*Stream)(s)))
}

func (n *ps2netNotifee) ClosedStream(s *ps.Stream) {
	n.not.ClosedStream(n.net, inet.Stream((*Stream)(s)))
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"log"

	"github.com/spf13/cobra"

	generator "github.com/argoproj/argo-cd/v2/hack/gen-resources/generators"
	"github.com/argoproj/argo-cd/v2/util/db"
	"github.com/argoproj/argo-cd/v2/util/settings"

	cmdutil "github.com/argoproj/argo-cd/v2/cmd/util"
	"github.com/argoproj/argo-cd/v2/hack/gen-resources/util"
	"github.com/argoproj/argo-cd/v2/util/cli"
)

const (
	cliName = "argocd-generator"
)

func init() {
	cobra.OnInitialize(initConfig)
}

func initConfig() {
	cli.SetLogFormat(cmdutil.LogFormat)
	cli.SetLogLevel(cmdutil.LogLevel)
}

// NewCommand returns a new instance of an argocd command
func NewCommand() *cobra.Command {

	var generateOpts util.GenerateOpts

	var command = &cobra.Command{
		Use:   cliName,
		Short: "Generator for argocd resources",
		Run: func(c *cobra.Command, args []string) {
			c.HelpFunc()(c, args)
		},
		DisableAutoGenTag: true,
	}

	command.AddCommand(NewGenerateCommand(&generateOpts))
	command.AddCommand(NewCleanCommand(&generateOpts))

	command.PersistentFlags().StringVar(&generateOpts.Namespace, "kube-namespace", "argocd", "Name of the namespace on which Argo agent should be installed [$KUBE_NAMESPACE]")
	return command
}

func NewGenerateCommand(opts *util.GenerateOpts) *cobra.Command {
	var file string
	var command = &cobra.Command{
		Use:   "generate [-f file]",
		Short: "Generate entities",
		Long:  "Generate entities",
		Run: func(c *cobra.Command, args []string) {
			log.Printf("Retrieve configuration from %s", file)
			err := util.Parse(opts, file)
			if err != nil {
				log.Fatalf("Failed to retrieve configuration, %v", err.Error())
			}
			argoClientSet := util.ConnectToK8sArgoClientSet()
			clientSet := util.ConnectToK8sClientSet()

			settingsMgr := settings.NewSettingsManager(context.TODO(), clientSet, opts.Namespace)
			argoDB := db.NewDB(opts.Namespace, settingsMgr, clientSet)

			pg := generator.NewProjectGenerator(argoClientSet)
			ag := generator.NewApplicationGenerator(argoClientSet, clientSet, argoDB)
			rg := generator.NewRepoGenerator(util.ConnectToK8sClientSet())
			cg := generator.NewClusterGenerator(argoDB, util.ConnectToK8sClientSet(), util.ConnectToK8sConfig())

			err = pg.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate projects, %v", err.Error())
			}
			err = rg.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate repositories, %v", err.Error())
			}
			err = cg.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate clusters, %v", err.Error())
			}
			err = ag.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate applications, %v", err.Error())
			}
		},
	}
	command.Flags().StringVarP(&file, "file", "f", "", "")
	return command
}

func NewCleanCommand(opts *util.GenerateOpts) *cobra.Command {
	var command = &cobra.Command{
		Use:   "clean",
		Short: "Clean entities",
		Long:  "Clean entities",
		Run: func(c *cobra.Command, args []string) {
			argoClientSet := util.ConnectToK8sArgoClientSet()
			clientSet := util.ConnectToK8sClientSet()
			settingsMgr := settings.NewSettingsManager(context.TODO(), clientSet, opts.Namespace)
			argoDB := db.NewDB(opts.Namespace, settingsMgr, clientSet)

			pg := generator.NewProjectGenerator(argoClientSet)
			ag := generator.NewApplicationGenerator(argoClientSet, clientSet, argoDB)
			cg := generator.NewClusterGenerator(argoDB, clientSet, util.ConnectToK8sConfig())
			rg := generator.NewRepoGenerator(clientSet)

			err := pg.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean projects, %v", err.Error())
			}
			err = ag.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean applications, %v", err.Error())
			}
			err = cg.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean clusters, %v", err.Error())
			}
			err = rg.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean repositores, %v", err.Error())
			}
		},
	}
	return command
}
<commit_msg>fix: typo in namespace param (#8770)<commit_after>package commands

import (
	"context"
	"log"

	"github.com/spf13/cobra"

	generator "github.com/argoproj/argo-cd/v2/hack/gen-resources/generators"
	"github.com/argoproj/argo-cd/v2/util/db"
	"github.com/argoproj/argo-cd/v2/util/settings"

	cmdutil "github.com/argoproj/argo-cd/v2/cmd/util"
	"github.com/argoproj/argo-cd/v2/hack/gen-resources/util"
	"github.com/argoproj/argo-cd/v2/util/cli"
)

const (
	cliName = "argocd-generator"
)

func init() {
	cobra.OnInitialize(initConfig)
}

func initConfig() {
	cli.SetLogFormat(cmdutil.LogFormat)
	cli.SetLogLevel(cmdutil.LogLevel)
}

// NewCommand returns a new instance of an argocd command
func NewCommand() *cobra.Command {

	var generateOpts util.GenerateOpts

	var command = &cobra.Command{
		Use:   cliName,
		Short: "Generator for argocd resources",
		Run: func(c *cobra.Command, args []string) {
			c.HelpFunc()(c, args)
		},
		DisableAutoGenTag: true,
	}

	command.AddCommand(NewGenerateCommand(&generateOpts))
	command.AddCommand(NewCleanCommand(&generateOpts))

	return command
}

func NewGenerateCommand(opts *util.GenerateOpts) *cobra.Command {
	var file string
	var command = &cobra.Command{
		Use:   "generate [-f file]",
		Short: "Generate entities",
		Long:  "Generate entities",
		Run: func(c *cobra.Command, args []string) {
			log.Printf("Retrieve configuration from %s", file)
			err := util.Parse(opts, file)
			if err != nil {
				log.Fatalf("Failed to retrieve configuration, %v", err.Error())
			}
			argoClientSet := util.ConnectToK8sArgoClientSet()
			clientSet := util.ConnectToK8sClientSet()

			settingsMgr := settings.NewSettingsManager(context.TODO(), clientSet, opts.Namespace)
			argoDB := db.NewDB(opts.Namespace, settingsMgr, clientSet)

			pg := generator.NewProjectGenerator(argoClientSet)
			ag := generator.NewApplicationGenerator(argoClientSet, clientSet, argoDB)
			rg := generator.NewRepoGenerator(util.ConnectToK8sClientSet())
			cg := generator.NewClusterGenerator(argoDB, util.ConnectToK8sClientSet(), util.ConnectToK8sConfig())

			err = pg.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate projects, %v", err.Error())
			}
			err = rg.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate repositories, %v", err.Error())
			}
			err = cg.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate clusters, %v", err.Error())
			}
			err = ag.Generate(opts)
			if err != nil {
				log.Fatalf("Failed to generate applications, %v", err.Error())
			}
		},
	}
	command.Flags().StringVarP(&file, "file", "f", "", "")
	return command
}

func NewCleanCommand(opts *util.GenerateOpts) *cobra.Command {
	var command = &cobra.Command{
		Use:   "clean",
		Short: "Clean entities",
		Long:  "Clean entities",
		Run: func(c *cobra.Command, args []string) {
			argoClientSet := util.ConnectToK8sArgoClientSet()
			clientSet := util.ConnectToK8sClientSet()
			settingsMgr := settings.NewSettingsManager(context.TODO(), clientSet, opts.Namespace)
			argoDB := db.NewDB(opts.Namespace, settingsMgr, clientSet)

			pg := generator.NewProjectGenerator(argoClientSet)
			ag := generator.NewApplicationGenerator(argoClientSet, clientSet, argoDB)
			cg := generator.NewClusterGenerator(argoDB, clientSet, util.ConnectToK8sConfig())
			rg := generator.NewRepoGenerator(clientSet)

			err := pg.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean projects, %v", err.Error())
			}
			err = ag.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean applications, %v", err.Error())
			}
			err = cg.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean clusters, %v", err.Error())
			}
			err = rg.Clean(opts)
			if err != nil {
				log.Fatalf("Failed to clean repositores, %v", err.Error())
			}
		},
	}
	command.PersistentFlags().StringVar(&opts.Namespace, "kube-namespace", "argocd", "Name of the namespace where argocd is running [$KUBE_NAMESPACE]")
	return command
}
<|endoftext|>


<commit_before>// +build js,wasm

package websocket

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"strings"
	"sync"
	"syscall/js"
	"time"
)

const (
	webSocketStateConnecting = 0
	webSocketStateOpen       = 1
	webSocketStateClosing    = 2
	webSocketStateClosed     = 3
)

var errConnectionClosed = errors.New("connection is closed")

// Conn implements net.Conn interface for WebSockets in js/wasm.
type Conn struct {
	js.Value
	messageHandler  *js.Func
	closeHandler    *js.Func
	errorHandler    *js.Func
	mut             sync.Mutex
	currDataMut     sync.RWMutex
	currData        bytes.Buffer
	closeOnce       sync.Once
	closeSignalOnce sync.Once
	closeSignal     chan struct{}
	dataSignal      chan struct{}
	localAddr       net.Addr
	remoteAddr      net.Addr
	firstErr        error // only read this _after_ observing that closeSignal has been closed.
}

// NewConn creates a Conn given a regular js/wasm WebSocket Conn.
func NewConn(raw js.Value) *Conn {
	conn := &Conn{
		Value:       raw,
		closeSignal: make(chan struct{}),
		dataSignal:  make(chan struct{}, 1),
		localAddr:   NewAddr("0.0.0.0:0"),
		remoteAddr:  getRemoteAddr(raw),
	}
	// Force the JavaScript WebSockets API to use the ArrayBuffer type for
	// incoming messages instead of the Blob type. This is better for us because
	// ArrayBuffer can be converted to []byte synchronously but Blob cannot.
	conn.Set("binaryType", "arraybuffer")
	conn.setUpHandlers()
	return conn
}

func (c *Conn) Read(b []byte) (int, error) {
	if err := c.checkOpen(); err != nil {
		return c.readAfterErr(b)
	}

	for {
		c.currDataMut.RLock()
		n, _ := c.currData.Read(b)
		c.currDataMut.RUnlock()

		if n != 0 {
			// Data was ready. Return the number of bytes read.
			return n, nil
		}

		// There is no data ready to be read. Wait for more data or for the
		// connection to be closed.
		select {
		case <-c.dataSignal:
			continue
		case <-c.closeSignal:
			return c.readAfterErr(b)
		}
	}
}

// readAfterError reads from c.currData. If there is no more data left it
// returns c.firstErr if non-nil and otherwise returns io.EOF.
func (c *Conn) readAfterErr(b []byte) (int, error) {
	c.currDataMut.RLock()
	n, err := c.currData.Read(b)
	c.currDataMut.RUnlock()
	if n == 0 {
		if c.firstErr != nil {
			return 0, c.firstErr
		} else {
			return 0, io.EOF
		}
	}
	return n, err
}

// checkOpen returns an error if the connection is not open. Otherwise, it
// returns nil.
func (c *Conn) checkOpen() error {
	state := c.Get("readyState").Int()
	switch state {
	case webSocketStateClosed, webSocketStateClosing:
		return errConnectionClosed
	}
	return nil
}

func (c *Conn) Write(b []byte) (n int, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = recoveredValueToError(e)
		}
	}()
	if err := c.checkOpen(); err != nil {
		return 0, err
	}
	uint8Array := js.Global().Get("Uint8Array").New(len(b))
	for i, bi := range b {
		uint8Array.SetIndex(i, bi)
	}
	c.Call("send", uint8Array.Get("buffer"))
	return len(b), nil
}

// Close closes the connection. Only the first call to Close will receive the
// close error, subsequent and concurrent calls will return nil.
// This method is thread-safe.
func (c *Conn) Close() error {
	c.closeOnce.Do(func() {
		c.Call("close")
		c.signalClose(nil)
		c.releaseHandlers()
	})
	return nil
}

func (c *Conn) signalClose(err error) {
	c.closeSignalOnce.Do(func() {
		c.firstErr = err
		close(c.closeSignal)
	})
}

func (c *Conn) releaseHandlers() {
	c.mut.Lock()
	defer c.mut.Unlock()
	if c.messageHandler != nil {
		c.Call("removeEventListener", "message", *c.messageHandler)
		c.messageHandler.Release()
		c.messageHandler = nil
	}
	if c.closeHandler != nil {
		c.Call("removeEventListener", "close", *c.closeHandler)
		c.closeHandler.Release()
		c.closeHandler = nil
	}
	if c.errorHandler != nil {
		c.Call("removeEventListener", "error", *c.errorHandler)
		c.errorHandler.Release()
		c.errorHandler = nil
	}
}

func (c *Conn) LocalAddr() net.Addr {
	return c.localAddr
}

func getRemoteAddr(val js.Value) net.Addr {
	rawURL := val.Get("url").String()
	withoutPrefix := strings.TrimPrefix(rawURL, "ws://")
	withoutSuffix := strings.TrimSuffix(withoutPrefix, "/")
	return NewAddr(withoutSuffix)
}

func (c *Conn) RemoteAddr() net.Addr {
	return c.remoteAddr
}

// TODO: Return os.ErrNoDeadline. For now we return nil because multiplexers
// don't handle the error correctly.
func (c *Conn) SetDeadline(t time.Time) error {
	return nil
}

func (c *Conn) SetReadDeadline(t time.Time) error {
	return nil
}

func (c *Conn) SetWriteDeadline(t time.Time) error {
	return nil
}

func (c *Conn) setUpHandlers() {
	c.mut.Lock()
	defer c.mut.Unlock()
	if c.messageHandler != nil {
		// Message handlers already created. Nothing to do.
		return
	}
	messageHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		arrayBuffer := args[0].Get("data")
		data := arrayBufferToBytes(arrayBuffer)
		c.currDataMut.Lock()
		if _, err := c.currData.Write(data); err != nil {
			c.currDataMut.Unlock()
			return err
		}
		c.currDataMut.Unlock()

		// Non-blocking signal
		select {
		case c.dataSignal <- struct{}{}:
		default:
		}

		return nil
	})
	c.messageHandler = &messageHandler
	c.Call("addEventListener", "message", messageHandler)

	closeHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		go func() {
			c.signalClose(errorEventToError(args[0]))
			c.releaseHandlers()
		}()
		return nil
	})
	c.closeHandler = &closeHandler
	c.Call("addEventListener", "close", closeHandler)

	errorHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		// Unfortunately, the "error" event doesn't appear to give us any useful
		// information. All we can do is close the connection.
		c.Close()
		return nil
	})
	c.errorHandler = &errorHandler
	c.Call("addEventListener", "error", errorHandler)
}

func (c *Conn) waitForOpen() error {
	openSignal := make(chan struct{})
	handler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		close(openSignal)
		return nil
	})
	defer c.Call("removeEventListener", "open", handler)
	defer handler.Release()
	c.Call("addEventListener", "open", handler)
	select {
	case <-openSignal:
		return nil
	case <-c.closeSignal:
		// c.closeSignal means there was an error when trying to open the
		// connection.
		return c.firstErr
	}
}

// arrayBufferToBytes converts a JavaScript ArrayBuffer to a slice of bytes.
func arrayBufferToBytes(buffer js.Value) []byte {
	view := js.Global().Get("Uint8Array").New(buffer)
	dataLen := view.Get("length").Int()
	data := make([]byte, dataLen)
	for i := 0; i < dataLen; i++ {
		data[i] = byte(view.Index(i).Int())
	}
	return data
}

func errorEventToError(val js.Value) error {
	var typ string
	if gotType := val.Get("type"); gotType != js.Undefined() {
		typ = gotType.String()
	} else {
		typ = val.Type().String()
	}
	var reason string
	if gotReason := val.Get("reason"); gotReason != js.Undefined() && gotReason.String() != "" {
		reason = gotReason.String()
	} else {
		code := val.Get("code")
		if code != js.Undefined() {
			switch code := code.Int(); code {
			case 1006:
				reason = "code 1006: connection unexpectedly closed"
			default:
				reason = fmt.Sprintf("unexpected code: %d", code)
			}
		}
	}
	return fmt.Errorf("JavaScript error: (%s) %s", typ, reason)
}

func recoveredValueToError(e interface{}) error {
	switch e := e.(type) {
	case error:
		return e
	default:
		return fmt.Errorf("recovered from unexpected panic: %T %s", e, e)
	}
}
<commit_msg>Handle connection errors before handling data.<commit_after>// +build js,wasm

package websocket

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"strings"
	"sync"
	"syscall/js"
	"time"
)

const (
	webSocketStateConnecting = 0
	webSocketStateOpen       = 1
	webSocketStateClosing    = 2
	webSocketStateClosed     = 3
)

var errConnectionClosed = errors.New("connection is closed")

// Conn implements net.Conn interface for WebSockets in js/wasm.
type Conn struct {
	js.Value
	messageHandler  *js.Func
	closeHandler    *js.Func
	errorHandler    *js.Func
	mut             sync.Mutex
	currDataMut     sync.RWMutex
	currData        bytes.Buffer
	closeOnce       sync.Once
	closeSignalOnce sync.Once
	closeSignal     chan struct{}
	dataSignal      chan struct{}
	localAddr       net.Addr
	remoteAddr      net.Addr
	firstErr        error // only read this _after_ observing that closeSignal has been closed.
}

// NewConn creates a Conn given a regular js/wasm WebSocket Conn.
func NewConn(raw js.Value) *Conn {
	conn := &Conn{
		Value:       raw,
		closeSignal: make(chan struct{}),
		dataSignal:  make(chan struct{}, 1),
		localAddr:   NewAddr("0.0.0.0:0"),
		remoteAddr:  getRemoteAddr(raw),
	}
	// Force the JavaScript WebSockets API to use the ArrayBuffer type for
	// incoming messages instead of the Blob type. This is better for us because
	// ArrayBuffer can be converted to []byte synchronously but Blob cannot.
	conn.Set("binaryType", "arraybuffer")
	conn.setUpHandlers()
	return conn
}

func (c *Conn) Read(b []byte) (int, error) {
	select {
	case <-c.closeSignal:
		c.readAfterErr(b)
	default:
	}

	for {
		c.currDataMut.RLock()
		n, _ := c.currData.Read(b)
		c.currDataMut.RUnlock()

		if n != 0 {
			// Data was ready. Return the number of bytes read.
			return n, nil
		}

		// There is no data ready to be read. Wait for more data or for the
		// connection to be closed.
		select {
		case <-c.dataSignal:
		case <-c.closeSignal:
			return c.readAfterErr(b)
		}
	}
}

// readAfterError reads from c.currData. If there is no more data left it
// returns c.firstErr if non-nil and otherwise returns io.EOF.
func (c *Conn) readAfterErr(b []byte) (int, error) {
	if c.firstErr != nil {
		return 0, c.firstErr
	}
	c.currDataMut.RLock()
	n, err := c.currData.Read(b)
	c.currDataMut.RUnlock()
	return n, err
}

// checkOpen returns an error if the connection is not open. Otherwise, it
// returns nil.
func (c *Conn) checkOpen() error {
	state := c.Get("readyState").Int()
	switch state {
	case webSocketStateClosed, webSocketStateClosing:
		return errConnectionClosed
	}
	return nil
}

func (c *Conn) Write(b []byte) (n int, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = recoveredValueToError(e)
		}
	}()
	if err := c.checkOpen(); err != nil {
		return 0, err
	}
	uint8Array := js.Global().Get("Uint8Array").New(len(b))
	for i, bi := range b {
		uint8Array.SetIndex(i, bi)
	}
	c.Call("send", uint8Array.Get("buffer"))
	return len(b), nil
}

// Close closes the connection. Only the first call to Close will receive the
// close error, subsequent and concurrent calls will return nil.
// This method is thread-safe.
func (c *Conn) Close() error {
	c.closeOnce.Do(func() {
		c.Call("close")
		c.signalClose(nil)
		c.releaseHandlers()
	})
	return nil
}

func (c *Conn) signalClose(err error) {
	c.closeSignalOnce.Do(func() {
		c.firstErr = err
		close(c.closeSignal)
	})
}

func (c *Conn) releaseHandlers() {
	c.mut.Lock()
	defer c.mut.Unlock()
	if c.messageHandler != nil {
		c.Call("removeEventListener", "message", *c.messageHandler)
		c.messageHandler.Release()
		c.messageHandler = nil
	}
	if c.closeHandler != nil {
		c.Call("removeEventListener", "close", *c.closeHandler)
		c.closeHandler.Release()
		c.closeHandler = nil
	}
	if c.errorHandler != nil {
		c.Call("removeEventListener", "error", *c.errorHandler)
		c.errorHandler.Release()
		c.errorHandler = nil
	}
}

func (c *Conn) LocalAddr() net.Addr {
	return c.localAddr
}

func getRemoteAddr(val js.Value) net.Addr {
	rawURL := val.Get("url").String()
	withoutPrefix := strings.TrimPrefix(rawURL, "ws://")
	withoutSuffix := strings.TrimSuffix(withoutPrefix, "/")
	return NewAddr(withoutSuffix)
}

func (c *Conn) RemoteAddr() net.Addr {
	return c.remoteAddr
}

// TODO: Return os.ErrNoDeadline. For now we return nil because multiplexers
// don't handle the error correctly.
func (c *Conn) SetDeadline(t time.Time) error {
	return nil
}

func (c *Conn) SetReadDeadline(t time.Time) error {
	return nil
}

func (c *Conn) SetWriteDeadline(t time.Time) error {
	return nil
}

func (c *Conn) setUpHandlers() {
	c.mut.Lock()
	defer c.mut.Unlock()
	if c.messageHandler != nil {
		// Message handlers already created. Nothing to do.
		return
	}
	messageHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		arrayBuffer := args[0].Get("data")
		data := arrayBufferToBytes(arrayBuffer)
		c.currDataMut.Lock()
		if _, err := c.currData.Write(data); err != nil {
			c.currDataMut.Unlock()
			return err
		}
		c.currDataMut.Unlock()

		// Non-blocking signal
		select {
		case c.dataSignal <- struct{}{}:
		default:
		}

		return nil
	})
	c.messageHandler = &messageHandler
	c.Call("addEventListener", "message", messageHandler)

	closeHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		go func() {
			c.signalClose(errorEventToError(args[0]))
			c.releaseHandlers()
		}()
		return nil
	})
	c.closeHandler = &closeHandler
	c.Call("addEventListener", "close", closeHandler)

	errorHandler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		// Unfortunately, the "error" event doesn't appear to give us any useful
		// information. All we can do is close the connection.
		c.Close()
		return nil
	})
	c.errorHandler = &errorHandler
	c.Call("addEventListener", "error", errorHandler)
}

func (c *Conn) waitForOpen() error {
	openSignal := make(chan struct{})
	handler := js.FuncOf(func(this js.Value, args []js.Value) interface{} {
		close(openSignal)
		return nil
	})
	defer c.Call("removeEventListener", "open", handler)
	defer handler.Release()
	c.Call("addEventListener", "open", handler)
	select {
	case <-openSignal:
		return nil
	case <-c.closeSignal:
		// c.closeSignal means there was an error when trying to open the
		// connection.
		return c.firstErr
	}
}

// arrayBufferToBytes converts a JavaScript ArrayBuffer to a slice of bytes.
func arrayBufferToBytes(buffer js.Value) []byte {
	view := js.Global().Get("Uint8Array").New(buffer)
	dataLen := view.Get("length").Int()
	data := make([]byte, dataLen)
	for i := 0; i < dataLen; i++ {
		data[i] = byte(view.Index(i).Int())
	}
	return data
}

func errorEventToError(val js.Value) error {
	var typ string
	if gotType := val.Get("type"); gotType != js.Undefined() {
		typ = gotType.String()
	} else {
		typ = val.Type().String()
	}
	var reason string
	if gotReason := val.Get("reason"); gotReason != js.Undefined() && gotReason.String() != "" {
		reason = gotReason.String()
	} else {
		code := val.Get("code")
		if code != js.Undefined() {
			switch code := code.Int(); code {
			case 1006:
				reason = "code 1006: connection unexpectedly closed"
			default:
				reason = fmt.Sprintf("unexpected code: %d", code)
			}
		}
	}
	return fmt.Errorf("JavaScript error: (%s) %s", typ, reason)
}

func recoveredValueToError(e interface{}) error {
	switch e := e.(type) {
	case error:
		return e
	default:
		return fmt.Errorf("recovered from unexpected panic: %T %s", e, e)
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

//获取指定目录下的所有文件，不进入下一级目录搜索，可以匹配后缀过滤。
func DoListDir(dirPth string, suffix string, f func(fileName string) error) error {
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	for _, fi := range dir {
		if fi.IsDir() { // 忽略目录
			continue
		}
		newFile := dirPth + PthSep + fi.Name()
		if f(newFile) != nil {
			return errors.New("user quit")
		}
	}
	return nil
}

//获取指定目录下的所有文件，不进入下一级目录搜索，可以匹配后缀过滤。
func DoListDirEx(dirPth string, suffix string, f func(fullpath string, fileName string) error) error {
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	for _, fi := range dir {
		if fi.IsDir() { // 忽略目录
			continue
		}
		newFile := dirPth + PthSep + fi.Name()
		if f(newFile, fi.Name()) != nil {
			return errors.New("user quit")
		}
	}
	return nil
}

//获取指定目录下的所有文件，不进入下一级目录搜索，可以匹配后缀过滤。
func ListDir(dirPth string, suffix string, ch chan<- string) error {
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	for _, fi := range dir {
		if fi.IsDir() { // 忽略目录
			continue
		}
		newFile := dirPth + PthSep + fi.Name()
		ch <- newFile
	}
	return nil
}

func DoWalkDir(dirPth, suffix string, f func(fileName string, isdir bool) error) error {
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	err := filepath.Walk(dirPth,
		func(filename string, fi os.FileInfo, err error) error { //遍历目录
			//if err != nil { //忽略错误
			// return err
			//}
			if fi.IsDir() { // 忽略目录
				f(filename, true)
				return nil
			}
			f(filename, false)
			return nil
		})
	return err
}

//获取指定目录及所有子目录下的所有文件，可以匹配后缀过滤。
func WalkDir(dirPth, suffix string, ch chan<- string) error {
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	err := filepath.Walk(dirPth,
		func(filename string, fi os.FileInfo, err error) error { //遍历目录
			//if err != nil { //忽略错误
			// return err
			//}
			if fi.IsDir() { // 忽略目录
				return nil
			}
			ch <- filename
			return nil
		})
	return err
}

func PathExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil || os.IsExist(err)
}

func PathExists2(filename string) (bool, error) {
	_, err := os.Stat(filename)
	if err == nil {
		return true, nil
	}

	if e, ok := err.(*os.PathError); ok && e.Error() == os.ErrNotExist.Error() {
		return false, nil
	}
	return false, err
}
<commit_msg>fix bug<commit_after>package utils

import (
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

//获取指定目录下的所有文件，不进入下一级目录搜索，可以匹配后缀过滤。
func DoListDir(dirPth string, suffix string, f func(fileName string) error) error {
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil { //忽略错误
		return nil
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	for _, fi := range dir {
		if fi.IsDir() { // 忽略目录
			continue
		}
		newFile := dirPth + PthSep + fi.Name()
		if f(newFile) != nil {
			return errors.New("user quit")
		}
	}
	return nil
}

//获取指定目录下的所有文件，不进入下一级目录搜索，可以匹配后缀过滤。
func DoListDirEx(dirPth string, suffix string, f func(fullpath string, fileName string) error) error {
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil { //忽略错误
		return nil
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	for _, fi := range dir {
		if fi.IsDir() { // 忽略目录
			continue
		}
		newFile := dirPth + PthSep + fi.Name()
		if f(newFile, fi.Name()) != nil {
			return errors.New("user quit")
		}
	}
	return nil
}

//获取指定目录下的所有文件，不进入下一级目录搜索，可以匹配后缀过滤。
func ListDir(dirPth string, suffix string, ch chan<- string) error {
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil { //忽略错误
		return nil
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	for _, fi := range dir {
		if fi.IsDir() { // 忽略目录
			continue
		}
		newFile := dirPth + PthSep + fi.Name()
		ch <- newFile
	}
	return nil
}

func DoWalkDir(dirPth, suffix string, f func(fileName string, isdir bool) error) error {
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	err := filepath.Walk(dirPth,
		func(filename string, fi os.FileInfo, err error) error { //遍历目录
			if err != nil { //忽略错误
				// return err
				return nil
			}
			if fi.IsDir() { // 忽略目录
				f(filename, true)
				return nil
			}
			f(filename, false)
			return nil
		})
	return err
}

//获取指定目录及所有子目录下的所有文件，可以匹配后缀过滤。
func WalkDir(dirPth, suffix string, ch chan<- string) error {
	suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写
	err := filepath.Walk(dirPth,
		func(filename string, fi os.FileInfo, err error) error { //遍历目录
			if err != nil { //忽略错误
				// return err
				return nil
			}
			if fi.IsDir() { // 忽略目录
				return nil
			}
			ch <- filename
			return nil
		})
	return err
}

func PathExists(filename string) bool {
	_, err := os.Stat(filename)
	return err == nil || os.IsExist(err)
}

func PathExists2(filename string) (bool, error) {
	_, err := os.Stat(filename)
	if err == nil {
		return true, nil
	}

	if e, ok := err.(*os.PathError); ok && e.Error() == os.ErrNotExist.Error() {
		return false, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>package utils

import (
	"io/ioutil"
	"net/http"
	"path/filepath"
	"strings"

	l4g "github.com/alecthomas/log4go"
	"github.com/cloudfoundry/jibber_jabber"
	"github.com/mattermost/platform/model"
	"github.com/nicksnyder/go-i18n/i18n"
)

var T i18n.TranslateFunc
var locales map[string]string = make(map[string]string)

func InitTranslations() {
	i18nDirectory := FindDir("i18n")
	files, _ := ioutil.ReadDir(i18nDirectory)
	for _, f := range files {
		if filepath.Ext(f.Name()) == ".json" {
			filename := f.Name()
			locales[strings.Split(filename, ".")[0]] = i18nDirectory + filename
			i18n.MustLoadTranslationFile(i18nDirectory + filename)
		}
	}

	T = GetTranslationsBySystemLocale()
}

func GetTranslationsBySystemLocale() i18n.TranslateFunc {
	locale := model.DEFAULT_LOCALE
	if userLanguage, err := jibber_jabber.DetectLanguage(); err == nil {
		if _, ok := locales[userLanguage]; ok {
			locale = userLanguage
		} else {
			l4g.Error("Failed to load system translations for '%v' attempting to fall back to '%v'", locale, model.DEFAULT_LOCALE)
			locale = model.DEFAULT_LOCALE
		}
	}

	if locales[locale] == "" {
		panic("Failed to load system translations for '" + model.DEFAULT_LOCALE + "'")
	}

	translations, _ := i18n.Tfunc(locale)
	if translations == nil {
		panic("Failed to load system translations")
	}

	l4g.Info(translations("utils.i18n.loaded"), locale, locales[locale])
	return translations
}

func GetUserTranslations(locale string) i18n.TranslateFunc {
	if _, ok := locales[locale]; !ok {
		locale = model.DEFAULT_LOCALE
	}

	translations, _ := i18n.Tfunc(locale)
	return translations
}

func SetTranslations(locale string) i18n.TranslateFunc {
	translations, _ := i18n.Tfunc(locale)
	return translations
}

func GetTranslationsAndLocale(w http.ResponseWriter, r *http.Request) (i18n.TranslateFunc, string) {
	headerLocale := strings.Split(strings.Split(r.Header.Get("Accept-Language"), ",")[0], "-")[0]
	if locales[headerLocale] != "" {
		translations, _ := i18n.Tfunc(headerLocale)
		return translations, headerLocale
	}

	translations, _ := i18n.Tfunc(model.DEFAULT_LOCALE)
	return translations, model.DEFAULT_LOCALE
}
<commit_msg>Adding english fallback to server<commit_after>package utils

import (
	"io/ioutil"
	"net/http"
	"path/filepath"
	"strings"

	l4g "github.com/alecthomas/log4go"
	"github.com/cloudfoundry/jibber_jabber"
	"github.com/mattermost/platform/model"
	"github.com/nicksnyder/go-i18n/i18n"
)

var T i18n.TranslateFunc
var locales map[string]string = make(map[string]string)

func InitTranslations() {
	i18nDirectory := FindDir("i18n")
	files, _ := ioutil.ReadDir(i18nDirectory)
	for _, f := range files {
		if filepath.Ext(f.Name()) == ".json" {
			filename := f.Name()
			locales[strings.Split(filename, ".")[0]] = i18nDirectory + filename
			i18n.MustLoadTranslationFile(i18nDirectory + filename)
		}
	}

	T = GetTranslationsBySystemLocale()
}

func GetTranslationsBySystemLocale() i18n.TranslateFunc {
	locale := model.DEFAULT_LOCALE
	if userLanguage, err := jibber_jabber.DetectLanguage(); err == nil {
		if _, ok := locales[userLanguage]; ok {
			locale = userLanguage
		} else {
			l4g.Error("Failed to load system translations for '%v' attempting to fall back to '%v'", locale, model.DEFAULT_LOCALE)
			locale = model.DEFAULT_LOCALE
		}
	}

	if locales[locale] == "" {
		panic("Failed to load system translations for '" + model.DEFAULT_LOCALE + "'")
	}

	translations, _ := TfuncWithFallback(locale)
	if translations == nil {
		panic("Failed to load system translations")
	}

	l4g.Info(translations("utils.i18n.loaded"), locale, locales[locale])
	return translations
}

func GetUserTranslations(locale string) i18n.TranslateFunc {
	if _, ok := locales[locale]; !ok {
		locale = model.DEFAULT_LOCALE
	}

	translations, _ := TfuncWithFallback(locale)
	return translations
}

func SetTranslations(locale string) i18n.TranslateFunc {
	translations, _ := TfuncWithFallback(locale)
	return translations
}

func GetTranslationsAndLocale(w http.ResponseWriter, r *http.Request) (i18n.TranslateFunc, string) {
	headerLocale := strings.Split(strings.Split(r.Header.Get("Accept-Language"), ",")[0], "-")[0]
	if locales[headerLocale] != "" {
		translations, _ := TfuncWithFallback(headerLocale)
		return translations, headerLocale
	}

	translations, _ := TfuncWithFallback(model.DEFAULT_LOCALE)
	return translations, model.DEFAULT_LOCALE
}

func TfuncWithFallback(pref string, prefs ...string) TranslateFunc {
	t := i18n.MustTfunc(pref, prefs...)
	return func(translationID string, args ...interface{}) string {
		if translated := t(translationID, args...); translated != translationID {
			return translated
		}
		return i18n.MustTfunc("en", args...)
	}
}
<|endoftext|>


<commit_before>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package api

import (
	"github.com/gin-gonic/gin"
	"github.com/jinzhu/gorm"
	"github.com/pufferpanel/pufferpanel/v2"
	"github.com/pufferpanel/pufferpanel/v2/panel/models"
	"github.com/pufferpanel/pufferpanel/v2/panel/services"
	"github.com/pufferpanel/pufferpanel/v2/panel/web/handlers"
	"github.com/pufferpanel/pufferpanel/v2/response"
	"github.com/pufferpanel/pufferpanel/v2/scope"
	"net/http"
)

func registerUsers(g *gin.RouterGroup) {
	g.Handle("GET", "", handlers.OAuth2Handler(scope.UsersView, false), searchUsers)
	g.Handle("OPTIONS", "", response.CreateOptions("GET"))

	g.Handle("PUT", "/:username", handlers.OAuth2Handler(scope.UsersEdit, false), createUser)
	g.Handle("GET", "/:username", handlers.OAuth2Handler(scope.UsersView, false), getUser)
	g.Handle("POST", "/:username", handlers.OAuth2Handler(scope.UsersEdit, false), updateUser)
	g.Handle("DELETE", "/:username", handlers.OAuth2Handler(scope.UsersEdit, false), deleteUser)
	g.Handle("OPTIONS", "/:username", response.CreateOptions("PUT", "GET", "POST", "DELETE"))

	g.Handle("GET", "/:username/perms", handlers.OAuth2Handler(scope.UsersView, false), response.NotImplemented)
	g.Handle("PUT", "/:username/perms", handlers.OAuth2Handler(scope.UsersEdit, false), response.NotImplemented)
	g.Handle("OPTIONS", "/:username/perms", response.CreateOptions("PUT", "GET"))
}

func searchUsers(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	search := newUserSearch()
	err = c.BindJSON(search)
	if response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if search.PageLimit > MaxPageSize {
		search.PageLimit = MaxPageSize
	}

	var results *models.Users
	var total uint
	if results, total, err = us.Search(search.Username, search.Email, search.PageLimit, search.Page); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.JSON(http.StatusOK, &UserSearchResponse{
		Users: models.FromUsers(results),
		Metadata: &response.Metadata{Paging: &response.Paging{
			Page:    search.Page,
			Size:    search.PageLimit,
			MaxSize: MaxPageSize,
			Total:   total,
		}},
	})
}

func createUser(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	var viewModel models.UserView
	if err = c.BindJSON(&viewModel); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}
	viewModel.Username = c.Param("username")

	if err = viewModel.Valid(false); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if viewModel.Password == "" {
		response.HandleError(c, pufferpanel.ErrFieldRequired("password"), http.StatusBadRequest)
		return
	}

	user := &models.User{}
	viewModel.CopyToModel(user)

	if err = us.Create(user); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

func getUser(c *gin.Context) {
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	username := c.Param("username")

	user, err := us.Get(username)
	if err != nil && gorm.IsRecordNotFoundError(err) {
		c.AbortWithStatus(http.StatusNotFound)
		return
	} else if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.JSON(http.StatusOK, models.FromUser(user))
}

func updateUser(c *gin.Context) {
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	username := c.Param("username")

	var viewModel models.UserView
	if err := c.BindJSON(&viewModel); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if err := viewModel.Valid(true); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	user, err := us.Get(username)
	if err != nil && gorm.IsRecordNotFoundError(err) {
		c.AbortWithStatus(http.StatusNotFound)
		return
	} else if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	viewModel.CopyToModel(user)

	if err = us.Update(user); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

func deleteUser(c *gin.Context) {
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	username := c.Param("username")

	user, err := us.Get(username)
	if err != nil && gorm.IsRecordNotFoundError(err) {
		c.AbortWithStatus(http.StatusNotFound)
		return
	} else if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	if err = us.Delete(user.Username); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

type UserSearch struct {
	Username  string `json:"username"`
	Email     string `json:"email"`
	PageLimit uint   `json:"limit"`
	Page      uint   `json:"page"`
}

type UserSearchResponse struct {
	Users []*models.UserView `json:"users"`
	*response.Metadata
}

func newUserSearch() *UserSearch {
	return &UserSearch{
		Username:  "*",
		Email:     "*",
		PageLimit: DefaultPageSize,
		Page:      1,
	}
}
<commit_msg>Fix search to use query string instead<commit_after>/*
 Copyright 2018 Padduck, LLC
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 	http://www.apache.org/licenses/LICENSE-2.0
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package api

import (
	"github.com/gin-gonic/gin"
	"github.com/jinzhu/gorm"
	"github.com/pufferpanel/pufferpanel/v2"
	"github.com/pufferpanel/pufferpanel/v2/panel/models"
	"github.com/pufferpanel/pufferpanel/v2/panel/services"
	"github.com/pufferpanel/pufferpanel/v2/panel/web/handlers"
	"github.com/pufferpanel/pufferpanel/v2/response"
	"github.com/pufferpanel/pufferpanel/v2/scope"
	"net/http"
)

func registerUsers(g *gin.RouterGroup) {
	g.Handle("GET", "", handlers.OAuth2Handler(scope.UsersView, false), searchUsers)
	g.Handle("OPTIONS", "", response.CreateOptions("GET"))

	g.Handle("PUT", "/:username", handlers.OAuth2Handler(scope.UsersEdit, false), createUser)
	g.Handle("GET", "/:username", handlers.OAuth2Handler(scope.UsersView, false), getUser)
	g.Handle("POST", "/:username", handlers.OAuth2Handler(scope.UsersEdit, false), updateUser)
	g.Handle("DELETE", "/:username", handlers.OAuth2Handler(scope.UsersEdit, false), deleteUser)
	g.Handle("OPTIONS", "/:username", response.CreateOptions("PUT", "GET", "POST", "DELETE"))

	g.Handle("GET", "/:username/perms", handlers.OAuth2Handler(scope.UsersView, false), response.NotImplemented)
	g.Handle("PUT", "/:username/perms", handlers.OAuth2Handler(scope.UsersEdit, false), response.NotImplemented)
	g.Handle("OPTIONS", "/:username/perms", response.CreateOptions("PUT", "GET"))
}

func searchUsers(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	search := newUserSearch()
	err = c.ShouldBind(search)
	if response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if search.PageLimit > MaxPageSize {
		search.PageLimit = MaxPageSize
	}

	var results *models.Users
	var total uint
	if results, total, err = us.Search(search.Username, search.Email, search.PageLimit, search.Page); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.JSON(http.StatusOK, &UserSearchResponse{
		Users: models.FromUsers(results),
		Metadata: &response.Metadata{Paging: &response.Paging{
			Page:    search.Page,
			Size:    search.PageLimit,
			MaxSize: MaxPageSize,
			Total:   total,
		}},
	})
}

func createUser(c *gin.Context) {
	var err error
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	var viewModel models.UserView
	if err = c.BindJSON(&viewModel); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}
	viewModel.Username = c.Param("username")

	if err = viewModel.Valid(false); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if viewModel.Password == "" {
		response.HandleError(c, pufferpanel.ErrFieldRequired("password"), http.StatusBadRequest)
		return
	}

	user := &models.User{}
	viewModel.CopyToModel(user)

	if err = us.Create(user); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

func getUser(c *gin.Context) {
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	username := c.Param("username")

	user, err := us.Get(username)
	if err != nil && gorm.IsRecordNotFoundError(err) {
		c.AbortWithStatus(http.StatusNotFound)
		return
	} else if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.JSON(http.StatusOK, models.FromUser(user))
}

func updateUser(c *gin.Context) {
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	username := c.Param("username")

	var viewModel models.UserView
	if err := c.BindJSON(&viewModel); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	if err := viewModel.Valid(true); response.HandleError(c, err, http.StatusBadRequest) {
		return
	}

	user, err := us.Get(username)
	if err != nil && gorm.IsRecordNotFoundError(err) {
		c.AbortWithStatus(http.StatusNotFound)
		return
	} else if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	viewModel.CopyToModel(user)

	if err = us.Update(user); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

func deleteUser(c *gin.Context) {
	db := handlers.GetDatabase(c)
	us := &services.User{DB: db}

	username := c.Param("username")

	user, err := us.Get(username)
	if err != nil && gorm.IsRecordNotFoundError(err) {
		c.AbortWithStatus(http.StatusNotFound)
		return
	} else if response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	if err = us.Delete(user.Username); response.HandleError(c, err, http.StatusInternalServerError) {
		return
	}

	c.Status(http.StatusNoContent)
}

type UserSearch struct {
	Username  string `form:"username"`
	Email     string `form:"email"`
	PageLimit uint   `form:"limit"`
	Page      uint   `form:"page"`
}

type UserSearchResponse struct {
	Users []*models.UserView `json:"users"`
	*response.Metadata
}

func newUserSearch() *UserSearch {
	return &UserSearch{
		Username:  "*",
		Email:     "*",
		PageLimit: DefaultPageSize,
		Page:      1,
	}
}
<|endoftext|>


<commit_before>package amazon

import (
	"errors"
	"strings"

	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/sirupsen/logrus"
)

func (c client) RegisterTaskDefinition(task *ecs.RegisterTaskDefinitionInput) (*string, error) {
	logrus.Debug("Register Task Definition")
	def, err := c.ECS.RegisterTaskDefinition(task)
	if err != nil {
		return nil, err
	}
	return def.TaskDefinition.TaskDefinitionArn, err
}

func (c client) CreateCluster() (*string, error) {
	logrus.Debug("Create cluster ", c.Cluster)
	response, err := c.ECS.CreateCluster(&ecs.CreateClusterInput{ClusterName: &c.Cluster})
	if err != nil {
		return nil, err
	}
	return response.Cluster.Status, nil
}

func (c client) DeleteCluster() error {
	logrus.Debug("Delete cluster ", c.Cluster)
	response, err := c.ECS.DeleteCluster(&ecs.DeleteClusterInput{Cluster: &c.Cluster})
	if err != nil {
		return err
	}
	if *response.Cluster.Status == "INACTIVE" {
		return nil
	}
	return errors.New("Failed to delete cluster, status: " + *response.Cluster.Status)
}

func (c client) ClusterExists() (bool, error) {
	logrus.Debug("Check if cluster was already created: ", c.Cluster)
	clusters, err := c.ECS.ListClusters(nil)
	if err != nil {
		return false, err
	}
	found := false
	for _, arn := range clusters.ClusterArns {
		found = found || strings.HasSuffix(*arn, "/"+c.Cluster)
	}
	return found, nil
}
<commit_msg>Use DescribeCluster as ListCluster is a Paginated API<commit_after>package amazon

import (
	"errors"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/sirupsen/logrus"
)

func (c client) RegisterTaskDefinition(task *ecs.RegisterTaskDefinitionInput) (*string, error) {
	logrus.Debug("Register Task Definition")
	def, err := c.ECS.RegisterTaskDefinition(task)
	if err != nil {
		return nil, err
	}
	return def.TaskDefinition.TaskDefinitionArn, err
}

func (c client) CreateCluster() (*string, error) {
	logrus.Debug("Create cluster ", c.Cluster)
	response, err := c.ECS.CreateCluster(&ecs.CreateClusterInput{ClusterName: &c.Cluster})
	if err != nil {
		return nil, err
	}
	return response.Cluster.Status, nil
}

func (c client) DeleteCluster() error {
	logrus.Debug("Delete cluster ", c.Cluster)
	response, err := c.ECS.DeleteCluster(&ecs.DeleteClusterInput{Cluster: &c.Cluster})
	if err != nil {
		return err
	}
	if *response.Cluster.Status == "INACTIVE" {
		return nil
	}
	return errors.New("Failed to delete cluster, status: " + *response.Cluster.Status)
}

func (c client) ClusterExists() (bool, error) {
	logrus.Debug("Check if cluster was already created: ", c.Cluster)
	clusters, err := c.ECS.DescribeClusters(&ecs.DescribeClustersInput{
		Clusters: []*string{aws.String(c.Cluster)},
	})
	if err != nil {
		return false, err
	}
	return len(clusters.Clusters) > 0, nil
}
<|endoftext|>


<commit_before>package main

import (
	"container/list"
	"flag"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/brnstz/routine/wikimg"
)

var (
	// Print an HTML div with the hex background
	fmtSpec = `<a style="text-decoration: none" href="%s"><div style="background: %s; width=100%%"> </div></a>`

	// cache is our global cache of urls to imgResponse values
	cache = newColorCache(50000)
)

// colorCache is a cache of recent URLs to imgResponse values. It expires older
// URLs once it contains the maximum number of values.
type colorCache struct {
	hmap  map[string]imgResponse
	count int
	max   int
	mutex sync.RWMutex
	exp   *list.List
}

// newColorCache creates colorCache that holds max items.
func newColorCache(max int) *colorCache {
	return &colorCache{
		hmap:  map[string]imgResponse{},
		count: 0,
		max:   max,
		mutex: sync.RWMutex{},
		exp:   list.New(),
	}
}

// Add saves a url and its response to our cache
func (cc *colorCache) Add(url string, resp imgResponse) {
	// Lock the cache while we're adding
	cc.mutex.Lock()

	if cc.count >= cc.max {
		// If we've exceeded the max size, we must remove the oldest
		// element

		// Find the last element
		back := cc.exp.Back()

		// Remove it from the cache
		delete(cc.hmap, back.Value.(string))

		// Also remove it from the exp list
		cc.exp.Remove(back)
	} else {

		// Otherwise, we didn't remove anything so increment count
		cc.count++
	}

	// Add new url to be last to expire
	cc.exp.PushFront(url)

	// Save its value
	cc.hmap[url] = resp

	// Done locking
	cc.mutex.Unlock()
}

// Get retrieves an imgResponse by its url, returning whether it was found or
// not as the second value
func (cc *colorCache) Get(url string) (imgResponse, bool) {
	cc.mutex.RLock()

	// Get it within read lock
	resp, ok := cc.hmap[url]

	cc.mutex.RUnlock()

	return resp, ok
}

// GetMulti feeds at most max values into the out channel, closing it when all
// possible entries have been exhausted (may be less than max)
func (cc *colorCache) GetMulti(max int, out chan imgResponse) {
	cc.mutex.RLock()

	i := 0
	for _, v := range cc.hmap {
		if i > max {
			break
		}

		// Skip results that were errors
		if v.err != nil {
			continue
		}

		i++
		out <- v
	}
	close(out)

	cc.mutex.RUnlock()
}

// imgRequest is a request to get the first color from a URL
type imgRequest struct {
	p         *wikimg.Puller
	url       string
	responses chan imgResponse
}

// imgResponse contains the result of processing an imgRequest
type imgResponse struct {
	url string
	hex string
	err error
}

// worker takes imgRequests on the in channel, processes them and sends
// an imgResponse back on the request's channel
func worker(in chan *imgRequest) {
	for req := range in {
		var resp imgResponse

		// Check cache first
		resp, ok := cache.Get(req.url)

		if !ok {

			// It wasn't in the cache, so actually get it and add it
			_, resp.hex, resp.err = req.p.FirstColor(req.url)
			resp.url = req.url

			cache.Add(req.url, resp)
		}

		// Send it back on our response channel
		req.responses <- resp
	}
}

func main() {
	var max, bgmax, workers, buffer, port int

	flag.IntVar(&max, "max", 300, "max number of images per HTTP request")
	flag.IntVar(&bgmax, "bgmax", 1000, "max images to pull on each background request")
	flag.IntVar(&workers, "workers", 25, "number of background workers")
	flag.IntVar(&buffer, "buffer", 10000, "size of buffered channels")
	flag.IntVar(&port, "port", 8000, "HTTP port to listen on")
	flag.Parse()

	// Create a buffered channel for communicating between image
	// puller loop and workers
	imgReqs := make(chan *imgRequest, buffer)

	// Create workers
	for i := 0; i < workers; i++ {
		go worker(imgReqs)
	}

	// Create background pull task
	go func() {

		// Loop forever
		for {

			// Create a new image puller with our bgmax
			p := wikimg.NewPuller(bgmax)

			// Since this is running in the background, we can have a much
			// longer timeout
			ctx, _ := context.WithTimeout(context.Background(), time.Minute*10)

			// Set puller's Cancel channel, so it will be closed when the
			// context times out
			p.Cancel = ctx.Done()

			// Create a channel for receiving responses in this background
			// process
			responses := make(chan imgResponse, max)

			// Loop to retrieve more images
			for {
				imgURL, err := p.Next()

				if err == wikimg.EndOfResults {
					// Break from loop when end of results is reached
					break

				} else if err != nil {
					// Send error on the response channel and continue
					responses <- imgResponse{err: err}
					continue
				}

				// Create request and send on the global channel
				imgReqs <- &imgRequest{
					p:         p,
					url:       imgURL,
					responses: responses,
				}
			}

			for i := 0; i < max; i++ {
				// Read a response from the channel
				resp := <-responses

				// If there's an error, just log it on the server
				if resp.err != nil {
					log.Println(resp.err)
					continue
				}
			}

			// Sleep for a bit until next iteration
			time.Sleep(30 * time.Minute)
		}

	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// Create a channel
		responses := make(chan imgResponse, max)

		// Everybody gets a goroutine!
		go cache.GetMulti(max, responses)

		for resp := range responses {
			fmt.Fprintf(w, fmtSpec, resp.url, resp.hex)
			fmt.Fprintln(w)
		}
	})

	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
<commit_msg>adding comment about link<commit_after>package main

import (
	"container/list"
	"flag"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/brnstz/routine/wikimg"
)

var (
	// Print an HTML div with the hex background, also make it link
	// to the image itself
	fmtSpec = `<a style="text-decoration: none" href="%s"><div style="background: %s; width=100%%"> </div></a>`

	// cache is our global cache of urls to imgResponse values
	cache = newColorCache(50000)
)

// colorCache is a cache of recent URLs to imgResponse values. It expires older
// URLs once it contains the maximum number of values.
type colorCache struct {
	hmap  map[string]imgResponse
	count int
	max   int
	mutex sync.RWMutex
	exp   *list.List
}

// newColorCache creates colorCache that holds max items.
func newColorCache(max int) *colorCache {
	return &colorCache{
		hmap:  map[string]imgResponse{},
		count: 0,
		max:   max,
		mutex: sync.RWMutex{},
		exp:   list.New(),
	}
}

// Add saves a url and its response to our cache
func (cc *colorCache) Add(url string, resp imgResponse) {
	// Lock the cache while we're adding
	cc.mutex.Lock()

	if cc.count >= cc.max {
		// If we've exceeded the max size, we must remove the oldest
		// element

		// Find the last element
		back := cc.exp.Back()

		// Remove it from the cache
		delete(cc.hmap, back.Value.(string))

		// Also remove it from the exp list
		cc.exp.Remove(back)
	} else {

		// Otherwise, we didn't remove anything so increment count
		cc.count++
	}

	// Add new url to be last to expire
	cc.exp.PushFront(url)

	// Save its value
	cc.hmap[url] = resp

	// Done locking
	cc.mutex.Unlock()
}

// Get retrieves an imgResponse by its url, returning whether it was found or
// not as the second value
func (cc *colorCache) Get(url string) (imgResponse, bool) {
	cc.mutex.RLock()

	// Get it within read lock
	resp, ok := cc.hmap[url]

	cc.mutex.RUnlock()

	return resp, ok
}

// GetMulti feeds at most max values into the out channel, closing it when all
// possible entries have been exhausted (may be less than max)
func (cc *colorCache) GetMulti(max int, out chan imgResponse) {
	cc.mutex.RLock()

	i := 0
	for _, v := range cc.hmap {
		if i > max {
			break
		}

		// Skip results that were errors
		if v.err != nil {
			continue
		}

		i++
		out <- v
	}
	close(out)

	cc.mutex.RUnlock()
}

// imgRequest is a request to get the first color from a URL
type imgRequest struct {
	p         *wikimg.Puller
	url       string
	responses chan imgResponse
}

// imgResponse contains the result of processing an imgRequest
type imgResponse struct {
	url string
	hex string
	err error
}

// worker takes imgRequests on the in channel, processes them and sends
// an imgResponse back on the request's channel
func worker(in chan *imgRequest) {
	for req := range in {
		var resp imgResponse

		// Check cache first
		resp, ok := cache.Get(req.url)

		if !ok {

			// It wasn't in the cache, so actually get it and add it
			_, resp.hex, resp.err = req.p.FirstColor(req.url)
			resp.url = req.url

			cache.Add(req.url, resp)
		}

		// Send it back on our response channel
		req.responses <- resp
	}
}

func main() {
	var max, bgmax, workers, buffer, port int

	flag.IntVar(&max, "max", 300, "max number of images per HTTP request")
	flag.IntVar(&bgmax, "bgmax", 1000, "max images to pull on each background request")
	flag.IntVar(&workers, "workers", 25, "number of background workers")
	flag.IntVar(&buffer, "buffer", 10000, "size of buffered channels")
	flag.IntVar(&port, "port", 8000, "HTTP port to listen on")
	flag.Parse()

	// Create a buffered channel for communicating between image
	// puller loop and workers
	imgReqs := make(chan *imgRequest, buffer)

	// Create workers
	for i := 0; i < workers; i++ {
		go worker(imgReqs)
	}

	// Create background pull task
	go func() {

		// Loop forever
		for {

			// Create a new image puller with our bgmax
			p := wikimg.NewPuller(bgmax)

			// Since this is running in the background, we can have a much
			// longer timeout
			ctx, _ := context.WithTimeout(context.Background(), time.Minute*10)

			// Set puller's Cancel channel, so it will be closed when the
			// context times out
			p.Cancel = ctx.Done()

			// Create a channel for receiving responses in this background
			// process
			responses := make(chan imgResponse, max)

			// Loop to retrieve more images
			for {
				imgURL, err := p.Next()

				if err == wikimg.EndOfResults {
					// Break from loop when end of results is reached
					break

				} else if err != nil {
					// Send error on the response channel and continue
					responses <- imgResponse{err: err}
					continue
				}

				// Create request and send on the global channel
				imgReqs <- &imgRequest{
					p:         p,
					url:       imgURL,
					responses: responses,
				}
			}

			for i := 0; i < max; i++ {
				// Read a response from the channel
				resp := <-responses

				// If there's an error, just log it on the server
				if resp.err != nil {
					log.Println(resp.err)
					continue
				}
			}

			// Sleep for a bit until next iteration
			time.Sleep(30 * time.Minute)
		}

	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// Create a channel
		responses := make(chan imgResponse, max)

		// Everybody gets a goroutine!
		go cache.GetMulti(max, responses)

		for resp := range responses {
			fmt.Fprintf(w, fmtSpec, resp.url, resp.hex)
			fmt.Fprintln(w)
		}
	})

	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
<|endoftext|>


<commit_before>package stringutils

import (
	"crypto/rand"
	"fmt"
	"math"
	r "math/rand"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode/utf16"
)

func GetAfter(src string, find string) (string, bool) {
	index := strings.Index(src, find)
	if index > -1 && index+len(find) < len(src) {
		return src[(index + len(find)):], true
	}
	return src, false
}

func GetBefore(src string, find string) (string, bool) {
	index := strings.Index(src, find)
	if index > -1 && index < len(src) {
		return src[:index], true
	}
	return src, false
}

func UnicodeDecode(text string) string {
	regex, err := regexp.Compile(`(\\u[a-fA-F0-9]{4})`)
	if err != nil {
		return text
	}

	text = regex.ReplaceAllStringFunc(text, func(match string) string {
		_txt := match[2:]
		char, err := strconv.ParseInt(_txt, 16, 32)
		if err != nil {
			return match
		}
		return string(rune(int(char)))
	})

	regex, err = regexp.Compile(`(&#[\d]{2,6})`)
	if err != nil {
		return text
	}

	text = regex.ReplaceAllStringFunc(text, func(match string) string {
		_txt := match[2:]
		char, err := strconv.ParseInt(_txt, 10, 32)
		if err != nil {
			return match
		}
		return string(rune(int(char)))
	})

	return text
}

func UnicodeEncode(str string) (js, html string) {
	rs := []rune(str)
	js = ""
	html = ""
	for _, r := range rs {
		rint := int(r)
		if rint < 128 {
			js += string(r)
			html += string(r)
		} else {
			js += `\u` + strconv.FormatInt(int64(rint), 16) // json
			html += `&#` + strconv.Itoa(int(r)) + ";"       // 网页
		}
	}
	fmt.Printf("JSON: %s\n", js)
	fmt.Printf("HTML: %s\n", html)
	return
}

//比较是否在切片中存在，不区分大小写
func ExistSliceFold(srcSlice []string, elem string) bool {
	for _, v := range srcSlice {
		if strings.EqualFold(v, elem) {
			return true
		}
	}

	return false
}

//比较是否在切片中存在,区分大小写
func ExistSlice(srcSlice []string, elem string) bool {
	for _, v := range srcSlice {
		if v == elem {
			return true
		}
	}

	return false
}

//字符串转换uint16
func StringToUTF16(s string) []uint16 {
	return utf16.Encode([]rune(s + "\x00"))
}

func Float64IsZero(s float64) bool {
	if math.Abs(s) < 0.0001 {
		return true
	}
	return false
}

//取子串
func GetBetweenStr(str, start, end string) string {
	n := strings.Index(str, start)
	if n == -1 {
		n = 0
	}
	str = string([]byte(str)[n:])
	m := strings.Index(str, end)
	if m == -1 {
		m = len(str)
	}
	str = string([]byte(str)[:m])
	return str
}

//取子串
func Substr(str string, start, length int) string {
	rs := []rune(str)
	rl := len(rs)
	end := 0

	if start < 0 {
		start = rl - 1 + start
	}
	end = start + length

	if start > end {
		start, end = end, start
	}

	if start < 0 {
		start = 0
	}
	if start > rl {
		start = rl
	}
	if end < 0 {
		end = 0
	}
	if end > rl {
		end = rl
	}

	return string(rs[start:end])
}

//删除slice中的元素
func RemoveSliceElement(val interface{}, index int) interface{} {

	if reflect.TypeOf(val).Kind() != reflect.Slice {
		fmt.Println("val类型非slice")
		return nil
	}

	s := reflect.ValueOf(val)
	if index < 0 || index >= s.Len() {
		fmt.Println("传入参数有误")
		return nil
	}

	prev := s.Index(index)
	for i := index + 1; i < s.Len(); i++ {
		value := s.Index(i)
		prev.Set(value)
		prev = value
	}

	return s.Slice(0, s.Len()-1).Interface()
}

// RandomCreateBytes generate random []byte by specify chars.
func RandomCreateBytes(n int, alphabets ...byte) []byte {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	var randby bool
	if num, err := rand.Read(bytes); num != n || err != nil {
		r.Seed(time.Now().UnixNano())
		randby = true
	}
	for i, b := range bytes {
		if len(alphabets) == 0 {
			if randby {
				bytes[i] = alphanum[r.Intn(len(alphanum))]
			} else {
				bytes[i] = alphanum[b%byte(len(alphanum))]
			}
		} else {
			if randby {
				bytes[i] = alphabets[r.Intn(len(alphabets))]
			} else {
				bytes[i] = alphabets[b%byte(len(alphabets))]
			}
		}
	}
	return bytes
}
<commit_msg>增加md5值封装<commit_after>package stringutils

import (
	"crypto/md5"
	"crypto/rand"
	"fmt"
	"io"
	"math"
	r "math/rand"
	"reflect"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode/utf16"
)

func GetAfter(src string, find string) (string, bool) {
	index := strings.Index(src, find)
	if index > -1 && index+len(find) < len(src) {
		return src[(index + len(find)):], true
	}
	return src, false
}

func GetBefore(src string, find string) (string, bool) {
	index := strings.Index(src, find)
	if index > -1 && index < len(src) {
		return src[:index], true
	}
	return src, false
}

func UnicodeDecode(text string) string {
	regex, err := regexp.Compile(`(\\u[a-fA-F0-9]{4})`)
	if err != nil {
		return text
	}

	text = regex.ReplaceAllStringFunc(text, func(match string) string {
		_txt := match[2:]
		char, err := strconv.ParseInt(_txt, 16, 32)
		if err != nil {
			return match
		}
		return string(rune(int(char)))
	})

	regex, err = regexp.Compile(`(&#[\d]{2,6})`)
	if err != nil {
		return text
	}

	text = regex.ReplaceAllStringFunc(text, func(match string) string {
		_txt := match[2:]
		char, err := strconv.ParseInt(_txt, 10, 32)
		if err != nil {
			return match
		}
		return string(rune(int(char)))
	})

	return text
}

func UnicodeEncode(str string) (js, html string) {
	rs := []rune(str)
	js = ""
	html = ""
	for _, r := range rs {
		rint := int(r)
		if rint < 128 {
			js += string(r)
			html += string(r)
		} else {
			js += `\u` + strconv.FormatInt(int64(rint), 16) // json
			html += `&#` + strconv.Itoa(int(r)) + ";"       // 网页
		}
	}
	fmt.Printf("JSON: %s\n", js)
	fmt.Printf("HTML: %s\n", html)
	return
}

//比较是否在切片中存在，不区分大小写
func ExistSliceFold(srcSlice []string, elem string) bool {
	for _, v := range srcSlice {
		if strings.EqualFold(v, elem) {
			return true
		}
	}

	return false
}

//比较是否在切片中存在,区分大小写
func ExistSlice(srcSlice []string, elem string) bool {
	for _, v := range srcSlice {
		if v == elem {
			return true
		}
	}

	return false
}

//字符串转换uint16
func StringToUTF16(s string) []uint16 {
	return utf16.Encode([]rune(s + "\x00"))
}

func Float64IsZero(s float64) bool {
	if math.Abs(s) < 0.0001 {
		return true
	}
	return false
}

//取子串
func GetBetweenStr(str, start, end string) string {
	n := strings.Index(str, start)
	if n == -1 {
		n = 0
	}
	str = string([]byte(str)[n:])
	m := strings.Index(str, end)
	if m == -1 {
		m = len(str)
	}
	str = string([]byte(str)[:m])
	return str
}

//取子串
func Substr(str string, start, length int) string {
	rs := []rune(str)
	rl := len(rs)
	end := 0

	if start < 0 {
		start = rl - 1 + start
	}
	end = start + length

	if start > end {
		start, end = end, start
	}

	if start < 0 {
		start = 0
	}
	if start > rl {
		start = rl
	}
	if end < 0 {
		end = 0
	}
	if end > rl {
		end = rl
	}

	return string(rs[start:end])
}

//删除slice中的元素
func RemoveSliceElement(val interface{}, index int) interface{} {

	if reflect.TypeOf(val).Kind() != reflect.Slice {
		fmt.Println("val类型非slice")
		return nil
	}

	s := reflect.ValueOf(val)
	if index < 0 || index >= s.Len() {
		fmt.Println("传入参数有误")
		return nil
	}

	prev := s.Index(index)
	for i := index + 1; i < s.Len(); i++ {
		value := s.Index(i)
		prev.Set(value)
		prev = value
	}

	return s.Slice(0, s.Len()-1).Interface()
}

// RandomCreateBytes generate random []byte by specify chars.
func RandomCreateBytes(n int, alphabets ...byte) []byte {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	var randby bool
	if num, err := rand.Read(bytes); num != n || err != nil {
		r.Seed(time.Now().UnixNano())
		randby = true
	}
	for i, b := range bytes {
		if len(alphabets) == 0 {
			if randby {
				bytes[i] = alphanum[r.Intn(len(alphanum))]
			} else {
				bytes[i] = alphanum[b%byte(len(alphanum))]
			}
		} else {
			if randby {
				bytes[i] = alphabets[r.Intn(len(alphabets))]
			} else {
				bytes[i] = alphabets[b%byte(len(alphabets))]
			}
		}
	}
	return bytes
}

func SumMd5(txtInput string) string {
	h := md5.New()
	io.WriteString(h, txtInput)
	return fmt.Sprintf("%x", h.Sum(nil))
}
<|endoftext|>


<commit_before>package pigae

import (
	"fmt"
	"net/http"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"appengine"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]
	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "c++", "cpp", -1)
	terms := SplitForSearching(q, true)

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)
	})

	if len(words) == 0 {
		words, typedLangs = typedLangs, nil
	}

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByWordsWithFavorites(c, words, typedLangs, userProfile.FavoriteLanguages, userProfile.SeeNonFavorite, numberMaxResults)
	if err != nil {
		return err
	}

	typedLangsSet := make(map[string]bool, len(typedLangs))
	for _, lang := range typedLangs {
		typedLangsSet[lang] = true
	}

	// Highlight matching impls :)
	matchingImplIDs, err := dao.searchImplIDs(c, words, typedLangs)
	if err != nil {
		return err
	}
	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
		for i := range idiom.Implementations {
			impl := &idiom.Implementations[i]
			implIDStr := fmt.Sprintf("%d", impl.Id)
			if matchingImplIDs[implIDStr] {
				impl.Deco.Matching = true
			}
			if typedLangsSet[impl.LanguageName] {
				impl.Deco.SearchedLang = true
			}
		}
	}

	normalizedQ := strings.Join(terms, " ")
	return listResults(w, r, normalizedQ, hits)
}

func listResults(w http.ResponseWriter, r *http.Request, q string, idioms []*Idiom) error {
	data := &SearchResultsFacade{
		PageMeta: PageMeta{
			PageTitle:   "Idioms for \"" + q + "\"",
			Toggles:     toggles,
			SearchQuery: q,
		},
		UserProfile: readUserProfile(r),
		Q:           q,
		Results:     idioms,
	}

	return templates.ExecuteTemplate(w, "page-list-results", data)
}

func searchRedirect(w http.ResponseWriter, r *http.Request) error {
	q := r.FormValue("q")
	if q == "" {
		// Small hack for own convenience: empty search -> homepage
		http.Redirect(w, r, "/", 301)
		return nil
	}
	q = strings.Replace(q, " ", "+", -1)

	http.Redirect(w, r, hostPrefix()+"/search/"+q, http.StatusMovedPermanently)
	return nil
}

func listByLanguage(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	langsStr := vars["langs"]
	langs := strings.Split(langsStr, "_")
	langs = MapStrings(langs, normLang)
	langs = RemoveEmptyStrings(langs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByLangs(c, langs, numberMaxResults)
	if err != nil {
		return err
	}

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
	}

	niceLangs := MapStrings(langs, printNiceLang)
	return listResults(w, r, fmt.Sprintf("Language=%v", niceLangs), hits)
}
<commit_msg>Search: fix crashes for a star or a space.<commit_after>package pigae

import (
	"fmt"
	"net/http"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"appengine"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]
	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "c++", "cpp", -1)
	terms := SplitForSearching(q, true)

	if len(terms) == 0 {
		// Search query is empty or illegible...
		redirURL := hostPrefix() + "/about#about-block-all-idioms"
		http.Redirect(w, r, redirURL, http.StatusFound)
		return nil
	}

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)
	})

	if len(words) == 0 {
		words, typedLangs = typedLangs, nil
	}

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByWordsWithFavorites(c, words, typedLangs, userProfile.FavoriteLanguages, userProfile.SeeNonFavorite, numberMaxResults)
	if err != nil {
		return err
	}

	typedLangsSet := make(map[string]bool, len(typedLangs))
	for _, lang := range typedLangs {
		typedLangsSet[lang] = true
	}

	// Highlight matching impls :)
	matchingImplIDs, err := dao.searchImplIDs(c, words, typedLangs)
	if err != nil {
		return err
	}
	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
		for i := range idiom.Implementations {
			impl := &idiom.Implementations[i]
			implIDStr := fmt.Sprintf("%d", impl.Id)
			if matchingImplIDs[implIDStr] {
				impl.Deco.Matching = true
			}
			if typedLangsSet[impl.LanguageName] {
				impl.Deco.SearchedLang = true
			}
		}
	}

	normalizedQ := strings.Join(terms, " ")
	return listResults(w, r, normalizedQ, hits)
}

func listResults(w http.ResponseWriter, r *http.Request, q string, idioms []*Idiom) error {
	data := &SearchResultsFacade{
		PageMeta: PageMeta{
			PageTitle:   "Idioms for \"" + q + "\"",
			Toggles:     toggles,
			SearchQuery: q,
		},
		UserProfile: readUserProfile(r),
		Q:           q,
		Results:     idioms,
	}

	return templates.ExecuteTemplate(w, "page-list-results", data)
}

func searchRedirect(w http.ResponseWriter, r *http.Request) error {
	q := r.FormValue("q")
	if q == "" {
		// Small hack for own convenience: empty search -> homepage
		http.Redirect(w, r, "/", 301)
		return nil
	}
	q = strings.Replace(q, " ", "+", -1)

	http.Redirect(w, r, hostPrefix()+"/search/"+q, http.StatusMovedPermanently)
	return nil
}

func listByLanguage(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	langsStr := vars["langs"]
	langs := strings.Split(langsStr, "_")
	langs = MapStrings(langs, normLang)
	langs = RemoveEmptyStrings(langs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByLangs(c, langs, numberMaxResults)
	if err != nil {
		return err
	}

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
	}

	niceLangs := MapStrings(langs, printNiceLang)
	return listResults(w, r, fmt.Sprintf("Language=%v", niceLangs), hits)
}
<|endoftext|>