<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Andrew Bonventre (andybons@gmail.com)
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package cli

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"text/tabwriter"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"

	"github.com/spf13/cobra"
)

var errMissingParams = errors.New("missing or invalid parameters")

// panicGuard wraps an errorless command into one wrapping panics into errors.
// This simplifies error handling for many commands for which more elaborate
// error handling isn't needed and would otherwise bloat the code.
//
// Deprecated: When introducing a new cobra.Command, simply return an error.
func panicGuard(cmdFn func(*cobra.Command, []string)) func(*cobra.Command, []string) error {
	return func(c *cobra.Command, args []string) (err error) {
		defer func() {
			if r := recover(); r != nil {
				err = fmt.Errorf("%v", r)
			}
		}()
		cmdFn(c, args)
		return nil
	}
}

// panicf is only to be used when wrapped through panicGuard, since the
// stack trace doesn't matter then.
func panicf(format string, args ...interface{}) {
	panic(fmt.Sprintf(format, args...))
}

// getJSON is a convenience wrapper around util.GetJSON that uses our Context to populate
// parts of the request.
func getJSON(hostport, path string, v interface{}) error {
	httpClient, err := cliContext.GetHTTPClient()
	if err != nil {
		return err
	}
	return util.GetJSON(httpClient, cliContext.HTTPRequestScheme(), hostport, path, v)
}

// startCmd starts a node by initializing the stores and joining
// the cluster.
var startCmd = &cobra.Command{
	Use:   "start",
	Short: "start a node",
	Long: `
Start a CockroachDB node, which will export data from one or more
storage devices, specified via --store flags.

If no cluster exists yet and this is the first node, no additional
flags are required. If the cluster already exists, and this node is
uninitialized, specify the --join flag to point to any healthy node
(or list of nodes) already part of the cluster.
`,
	Example:      `  cockroach start --insecure --store=attrs=ssd,path=/mnt/ssd1 [--join=host:port,[host:port]]`,
	SilenceUsage: true,
	RunE:         runStart,
}

func initCacheSize() {
	if !cacheSize.isSet {
		if size, err := server.GetTotalMemory(); err == nil {
			cliContext.CacheSize = size / 2
		}
	}
}

func initInsecure() error {
	if !cliContext.Insecure || insecure.isSet {
		return nil
	}
	// The --insecure flag was not specified on the command line, verify that the
	// host refers to a loopback address.
	if connHost != "" {
		addr, err := net.ResolveIPAddr("ip", connHost)
		if err != nil {
			return err
		}
		if !addr.IP.IsLoopback() {
			return fmt.Errorf("specify --insecure to listen on external address %s", connHost)
		}
	} else if connHost == "" {
		cliContext.Addr = net.JoinHostPort("localhost", connPort)
		cliContext.HTTPAddr = net.JoinHostPort("localhost", httpPort)
	}
	return nil
}

// runStart starts the cockroach node using --store as the list of
// storage devices ("stores") on this machine and --join as the list
// of other active nodes used to join this node to the cockroach
// cluster, if this is its first time connecting.
func runStart(_ *cobra.Command, _ []string) error {
	initCacheSize()
	if err := initInsecure(); err != nil {
		return err
	}

	// Default the log directory to the the "logs" subdirectory of the first
	// non-memory store. We only do this for the "start" command which is why
	// this work occurs here and not in an OnInitialize function.
	f := flag.Lookup("log-dir")
	if !log.DirSet() {
		for _, spec := range cliContext.Stores.Specs {
			if spec.InMemory {
				continue
			}
			if err := f.Value.Set(filepath.Join(spec.Path, "logs")); err != nil {
				return err
			}
			break
		}
	}

	// Make sure the path exists
	if err := os.MkdirAll(f.Value.String(), 0755); err != nil {
		return err
	}

	info := util.GetBuildInfo()
	log.Infof("[build] %s @ %s (%s)", info.Tag, info.Time, info.Vers)

	// Default user for servers.
	cliContext.User = security.NodeUser

	stopper := stop.NewStopper()
	if err := cliContext.InitStores(stopper); err != nil {
		return fmt.Errorf("failed to initialize stores: %s", err)
	}

	if err := cliContext.InitNode(); err != nil {
		return fmt.Errorf("failed to initialize node: %s", err)
	}

	log.Info("starting cockroach node")
	s, err := server.NewServer(&cliContext.Context, stopper)
	if err != nil {
		return fmt.Errorf("failed to start Cockroach server: %s", err)
	}

	// We don't do this in NewServer since we don't want it in tests.
	if err := s.SetupReportingURLs(); err != nil {
		return err
	}

	if err := s.Start(); err != nil {
		return fmt.Errorf("cockroach server exited with error: %s", err)
	}

	pgURL, err := cliContext.PGURL(connUser)
	if err != nil {
		return err
	}

	tw := tabwriter.NewWriter(os.Stdout, 2, 1, 2, ' ', 0)
	fmt.Fprintf(tw, "build:\t%s @ %s (%s)\n", info.Tag, info.Time, info.Vers)
	fmt.Fprintf(tw, "admin:\t%s\n", cliContext.AdminURL())
	fmt.Fprintf(tw, "sql:\t%s\n", pgURL)
	if len(cliContext.SocketFile) != 0 {
		fmt.Fprintf(tw, "socket:\t%s\n", cliContext.SocketFile)
	}
	fmt.Fprintf(tw, "logs:\t%s\n", flag.Lookup("log-dir").Value)
	for i, spec := range cliContext.Stores.Specs {
		fmt.Fprintf(tw, "store[%d]:\t%s\n", i, spec)
	}
	if err := tw.Flush(); err != nil {
		return err
	}

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt, os.Kill)
	// TODO(spencer): move this behind a build tag.
	signal.Notify(signalCh, syscall.SIGTERM)

	// Block until one of the signals above is received or the stopper
	// is stopped externally (for example, via the quit endpoint).
	select {
	case <-stopper.ShouldStop():
	case <-signalCh:
		go s.Stop()
	}

	const msgDrain = "initiating graceful shutdown of server"
	log.Info(msgDrain)
	fmt.Fprintln(os.Stdout, msgDrain)

	go func() {
		ticker := time.NewTicker(5 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				if log.V(1) {
					log.Infof("running tasks:\n%s", stopper.RunningTasks())
				}
				log.Infof("%d running tasks", stopper.NumTasks())
			case <-stopper.ShouldStop():
				return
			}
		}
	}()

	select {
	case <-signalCh:
		log.Errorf("second signal received, initiating hard shutdown")
	case <-time.After(time.Minute):
		log.Errorf("time limit reached, initiating hard shutdown")
	case <-stopper.IsStopped():
		const msgDone = "server drained and shutdown completed"
		log.Infof(msgDone)
		fmt.Fprintln(os.Stdout, msgDone)
	}
	log.Flush()
	return nil
}

// exterminateCmd command shuts down the node server and
// destroys all data held by the node.
var exterminateCmd = &cobra.Command{
	Use:   "exterminate",
	Short: "destroy all data held by the node",
	Long: `
First shuts down the system and then destroys all data held by the
node, cycling through each store specified by --store flags.
`,
	SilenceUsage: true,
	RunE:         runExterminate,
}

// runExterminate destroys the data held in the specified stores.
func runExterminate(_ *cobra.Command, _ []string) error {
	stopper := stop.NewStopper()
	defer stopper.Stop()
	if err := cliContext.InitStores(stopper); err != nil {
		return util.Errorf("failed to initialize context: %s", err)
	}

	if err := runQuit(nil, nil); err != nil {
		return util.Errorf("shutdown node error: %s", err)
	}

	// Exterminate all data held in specified stores.
	for _, e := range cliContext.Engines {
		if rocksdb, ok := e.(*engine.RocksDB); ok {
			log.Infof("exterminating data from store %s", e)
			if err := rocksdb.Destroy(); err != nil {
				return util.Errorf("unable to destroy store %s: %s", e, err)
			}
		}
	}
	log.Infof("exterminated all data from stores %s", cliContext.Engines)
	return nil
}

// quitCmd command shuts down the node server.
var quitCmd = &cobra.Command{
	Use:   "quit",
	Short: "drain and shutdown node\n",
	Long: `
Shutdown the server. The first stage is drain, where any new requests
will be ignored by the server. When all extant requests have been
completed, the server exits.
`,
	SilenceUsage: true,
	RunE:         runQuit,
}

// runQuit accesses the quit shutdown path.
func runQuit(_ *cobra.Command, _ []string) error {
	admin, err := client.NewAdminClient(&cliContext.Context.Context, cliContext.HTTPAddr, client.Quit)
	if err != nil {
		return err
	}
	body, err := admin.Get()
	// TODO(tschottdorf): needs cleanup. An error here can happen if the shutdown
	// happened faster than the HTTP request made it back.
	if err != nil {
		return err
	}
	fmt.Printf("node drained and shutdown: %s\n", body)
	return nil
}
<commit_msg>cli: remove unnecessary conditional<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Andrew Bonventre (andybons@gmail.com)
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package cli

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"text/tabwriter"
	"time"

	"github.com/cockroachdb/cockroach/client"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/stop"

	"github.com/spf13/cobra"
)

var errMissingParams = errors.New("missing or invalid parameters")

// panicGuard wraps an errorless command into one wrapping panics into errors.
// This simplifies error handling for many commands for which more elaborate
// error handling isn't needed and would otherwise bloat the code.
//
// Deprecated: When introducing a new cobra.Command, simply return an error.
func panicGuard(cmdFn func(*cobra.Command, []string)) func(*cobra.Command, []string) error {
	return func(c *cobra.Command, args []string) (err error) {
		defer func() {
			if r := recover(); r != nil {
				err = fmt.Errorf("%v", r)
			}
		}()
		cmdFn(c, args)
		return nil
	}
}

// panicf is only to be used when wrapped through panicGuard, since the
// stack trace doesn't matter then.
func panicf(format string, args ...interface{}) {
	panic(fmt.Sprintf(format, args...))
}

// getJSON is a convenience wrapper around util.GetJSON that uses our Context to populate
// parts of the request.
func getJSON(hostport, path string, v interface{}) error {
	httpClient, err := cliContext.GetHTTPClient()
	if err != nil {
		return err
	}
	return util.GetJSON(httpClient, cliContext.HTTPRequestScheme(), hostport, path, v)
}

// startCmd starts a node by initializing the stores and joining
// the cluster.
var startCmd = &cobra.Command{
	Use:   "start",
	Short: "start a node",
	Long: `
Start a CockroachDB node, which will export data from one or more
storage devices, specified via --store flags.

If no cluster exists yet and this is the first node, no additional
flags are required. If the cluster already exists, and this node is
uninitialized, specify the --join flag to point to any healthy node
(or list of nodes) already part of the cluster.
`,
	Example:      `  cockroach start --insecure --store=attrs=ssd,path=/mnt/ssd1 [--join=host:port,[host:port]]`,
	SilenceUsage: true,
	RunE:         runStart,
}

func initCacheSize() {
	if !cacheSize.isSet {
		if size, err := server.GetTotalMemory(); err == nil {
			cliContext.CacheSize = size / 2
		}
	}
}

func initInsecure() error {
	if !cliContext.Insecure || insecure.isSet {
		return nil
	}
	// The --insecure flag was not specified on the command line, verify that the
	// host refers to a loopback address.
	if connHost != "" {
		addr, err := net.ResolveIPAddr("ip", connHost)
		if err != nil {
			return err
		}
		if !addr.IP.IsLoopback() {
			return fmt.Errorf("specify --insecure to listen on external address %s", connHost)
		}
	} else {
		cliContext.Addr = net.JoinHostPort("localhost", connPort)
		cliContext.HTTPAddr = net.JoinHostPort("localhost", httpPort)
	}
	return nil
}

// runStart starts the cockroach node using --store as the list of
// storage devices ("stores") on this machine and --join as the list
// of other active nodes used to join this node to the cockroach
// cluster, if this is its first time connecting.
func runStart(_ *cobra.Command, _ []string) error {
	initCacheSize()
	if err := initInsecure(); err != nil {
		return err
	}

	// Default the log directory to the the "logs" subdirectory of the first
	// non-memory store. We only do this for the "start" command which is why
	// this work occurs here and not in an OnInitialize function.
	f := flag.Lookup("log-dir")
	if !log.DirSet() {
		for _, spec := range cliContext.Stores.Specs {
			if spec.InMemory {
				continue
			}
			if err := f.Value.Set(filepath.Join(spec.Path, "logs")); err != nil {
				return err
			}
			break
		}
	}

	// Make sure the path exists
	if err := os.MkdirAll(f.Value.String(), 0755); err != nil {
		return err
	}

	info := util.GetBuildInfo()
	log.Infof("[build] %s @ %s (%s)", info.Tag, info.Time, info.Vers)

	// Default user for servers.
	cliContext.User = security.NodeUser

	stopper := stop.NewStopper()
	if err := cliContext.InitStores(stopper); err != nil {
		return fmt.Errorf("failed to initialize stores: %s", err)
	}

	if err := cliContext.InitNode(); err != nil {
		return fmt.Errorf("failed to initialize node: %s", err)
	}

	log.Info("starting cockroach node")
	s, err := server.NewServer(&cliContext.Context, stopper)
	if err != nil {
		return fmt.Errorf("failed to start Cockroach server: %s", err)
	}

	// We don't do this in NewServer since we don't want it in tests.
	if err := s.SetupReportingURLs(); err != nil {
		return err
	}

	if err := s.Start(); err != nil {
		return fmt.Errorf("cockroach server exited with error: %s", err)
	}

	pgURL, err := cliContext.PGURL(connUser)
	if err != nil {
		return err
	}

	tw := tabwriter.NewWriter(os.Stdout, 2, 1, 2, ' ', 0)
	fmt.Fprintf(tw, "build:\t%s @ %s (%s)\n", info.Tag, info.Time, info.Vers)
	fmt.Fprintf(tw, "admin:\t%s\n", cliContext.AdminURL())
	fmt.Fprintf(tw, "sql:\t%s\n", pgURL)
	if len(cliContext.SocketFile) != 0 {
		fmt.Fprintf(tw, "socket:\t%s\n", cliContext.SocketFile)
	}
	fmt.Fprintf(tw, "logs:\t%s\n", flag.Lookup("log-dir").Value)
	for i, spec := range cliContext.Stores.Specs {
		fmt.Fprintf(tw, "store[%d]:\t%s\n", i, spec)
	}
	if err := tw.Flush(); err != nil {
		return err
	}

	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt, os.Kill)
	// TODO(spencer): move this behind a build tag.
	signal.Notify(signalCh, syscall.SIGTERM)

	// Block until one of the signals above is received or the stopper
	// is stopped externally (for example, via the quit endpoint).
	select {
	case <-stopper.ShouldStop():
	case <-signalCh:
		go s.Stop()
	}

	const msgDrain = "initiating graceful shutdown of server"
	log.Info(msgDrain)
	fmt.Fprintln(os.Stdout, msgDrain)

	go func() {
		ticker := time.NewTicker(5 * time.Second)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				if log.V(1) {
					log.Infof("running tasks:\n%s", stopper.RunningTasks())
				}
				log.Infof("%d running tasks", stopper.NumTasks())
			case <-stopper.ShouldStop():
				return
			}
		}
	}()

	select {
	case <-signalCh:
		log.Errorf("second signal received, initiating hard shutdown")
	case <-time.After(time.Minute):
		log.Errorf("time limit reached, initiating hard shutdown")
	case <-stopper.IsStopped():
		const msgDone = "server drained and shutdown completed"
		log.Infof(msgDone)
		fmt.Fprintln(os.Stdout, msgDone)
	}
	log.Flush()
	return nil
}

// exterminateCmd command shuts down the node server and
// destroys all data held by the node.
var exterminateCmd = &cobra.Command{
	Use:   "exterminate",
	Short: "destroy all data held by the node",
	Long: `
First shuts down the system and then destroys all data held by the
node, cycling through each store specified by --store flags.
`,
	SilenceUsage: true,
	RunE:         runExterminate,
}

// runExterminate destroys the data held in the specified stores.
func runExterminate(_ *cobra.Command, _ []string) error {
	stopper := stop.NewStopper()
	defer stopper.Stop()
	if err := cliContext.InitStores(stopper); err != nil {
		return util.Errorf("failed to initialize context: %s", err)
	}

	if err := runQuit(nil, nil); err != nil {
		return util.Errorf("shutdown node error: %s", err)
	}

	// Exterminate all data held in specified stores.
	for _, e := range cliContext.Engines {
		if rocksdb, ok := e.(*engine.RocksDB); ok {
			log.Infof("exterminating data from store %s", e)
			if err := rocksdb.Destroy(); err != nil {
				return util.Errorf("unable to destroy store %s: %s", e, err)
			}
		}
	}
	log.Infof("exterminated all data from stores %s", cliContext.Engines)
	return nil
}

// quitCmd command shuts down the node server.
var quitCmd = &cobra.Command{
	Use:   "quit",
	Short: "drain and shutdown node\n",
	Long: `
Shutdown the server. The first stage is drain, where any new requests
will be ignored by the server. When all extant requests have been
completed, the server exits.
`,
	SilenceUsage: true,
	RunE:         runQuit,
}

// runQuit accesses the quit shutdown path.
func runQuit(_ *cobra.Command, _ []string) error {
	admin, err := client.NewAdminClient(&cliContext.Context.Context, cliContext.HTTPAddr, client.Quit)
	if err != nil {
		return err
	}
	body, err := admin.Get()
	// TODO(tschottdorf): needs cleanup. An error here can happen if the shutdown
	// happened faster than the HTTP request made it back.
	if err != nil {
		return err
	}
	fmt.Printf("node drained and shutdown: %s\n", body)
	return nil
}
<|endoftext|>


<commit_before>package lookup

import (
	"koding/db/models"
	"koding/db/mongodb"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

// MachineDocument represents a single MongodDB document from the jMachines
// collection.
type MachineDocument struct {
	Id          bson.ObjectId `bson:"_id" json:"-"`
	Label       string        `bson:"label"`
	Domain      string        `bson:"domain"`
	QueryString string        `bson:"queryString"`
	IpAddress   string        `bson:"ipAddress"`
	Assignee    struct {
		InProgress bool      `bson:"inProgress"`
		AssignedAt time.Time `bson:"assignedAt"`
	} `bson:"assignee"`
	Status struct {
		State      string    `bson:"state"`
		Reason     string    `bson:"reason"`
		ModifiedAt time.Time `bson:"modifiedAt"`
	} `bson:"status"`
	Provider   string               `bson:"provider"`
	Credential string               `bson:"credential"`
	CreatedAt  time.Time            `bson:"createdAt"`
	Meta       bson.M               `bson:"meta"`
	Users      []models.Permissions `bson:"users"`
	Groups     []models.Permissions `bson:"groups"`
}

type MongoDB struct {
	DB *mongodb.MongoDB
}

func NewMongoDB(url string) *MongoDB {
	return &MongoDB{
		DB: mongodb.NewMongoDB(url),
	}
}

// Iter iterates over all machine documents and executes fn for each new
// iteration.
func (m *MongoDB) Iter(fn func(MachineDocument)) error {
	query := func(c *mgo.Collection) error {
		machinesWithIds := bson.M{
			"meta.instanceId": bson.M{"$exists": true, "$ne": ""},
		}

		machine := MachineDocument{}
		iter := c.Find(machinesWithIds).Batch(150).Iter()
		for iter.Next(&machine) {
			fn(machine)
		}

		return iter.Close()
	}

	return m.DB.Run("jMachines", query)
}

// AlwaysOn returns all alwaysOn Machines
func (m *MongoDB) AlwaysOn() ([]MachineDocument, error) {
	machines := make([]MachineDocument, 0)

	query := func(c *mgo.Collection) error {
		alwaysOn := bson.M{
			"meta.alwaysOn": true,
		}

		machine := MachineDocument{}
		iter := c.Find(alwaysOn).Batch(150).Iter()
		for iter.Next(&machine) {
			machines = append(machines, machine)
		}

		return iter.Close()
	}

	if err := m.DB.Run("jMachines", query); err != nil {
		return nil, err
	}

	return machines, nil
}

// Machines returns a list of machines for the given instanceIds
func (m *MongoDB) Machines(instanceIds ...string) ([]MachineDocument, error) {
	machines := make([]MachineDocument, 0)

	query := func(c *mgo.Collection) error {
		all := bson.M{
			"meta.instanceId": bson.M{"$in": instanceIds},
		}

		machine := MachineDocument{}
		iter := c.Find(all).Batch(150).Iter()
		for iter.Next(&machine) {
			machines = append(machines, machine)
		}

		return iter.Close()
	}

	if err := m.DB.Run("jMachines", query); err != nil {
		return nil, err
	}

	return machines, nil
}

// Accounts returns a list of accounts for the give objectIds in non hex form
func (m *MongoDB) Accounts(ids ...string) ([]models.Account, error) {
	b := make([]bson.ObjectId, len(ids))
	for i, id := range ids {
		b[i] = bson.ObjectIdHex(id)
	}

	accounts := make([]models.Account, 0)

	query := func(c *mgo.Collection) error {
		all := bson.M{
			"_id": bson.M{"$in": b},
		}

		account := models.Account{}
		iter := c.Find(all).Batch(150).Iter()
		for iter.Next(&account) {
			accounts = append(accounts, account)
		}

		return iter.Close()
	}

	if err := m.DB.Run("jAccounts", query); err != nil {
		return nil, err
	}

	return accounts, nil
}

// RemoveAlwaysOn removes the alwaysOn flag for the given usernames
func (m *MongoDB) RemoveAlwaysOn(usernames ...string) error {
	query := func(c *mgo.Collection) error {
		_, err := c.UpdateAll(
			bson.M{"credential": bson.M{"$in": usernames}},
			bson.M{"$set": bson.M{"meta.alwaysOn": false}},
		)

		return err
	}

	return m.DB.Run("jMachines", query)
}
<commit_msg>cleaner: pick only koding provider<commit_after>package lookup

import (
	"koding/db/models"
	"koding/db/mongodb"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

// MachineDocument represents a single MongodDB document from the jMachines
// collection.
type MachineDocument struct {
	Id          bson.ObjectId `bson:"_id" json:"-"`
	Label       string        `bson:"label"`
	Domain      string        `bson:"domain"`
	QueryString string        `bson:"queryString"`
	IpAddress   string        `bson:"ipAddress"`
	Assignee    struct {
		InProgress bool      `bson:"inProgress"`
		AssignedAt time.Time `bson:"assignedAt"`
	} `bson:"assignee"`
	Status struct {
		State      string    `bson:"state"`
		Reason     string    `bson:"reason"`
		ModifiedAt time.Time `bson:"modifiedAt"`
	} `bson:"status"`
	Provider   string               `bson:"provider"`
	Credential string               `bson:"credential"`
	CreatedAt  time.Time            `bson:"createdAt"`
	Meta       bson.M               `bson:"meta"`
	Users      []models.Permissions `bson:"users"`
	Groups     []models.Permissions `bson:"groups"`
}

type MongoDB struct {
	DB *mongodb.MongoDB
}

func NewMongoDB(url string) *MongoDB {
	return &MongoDB{
		DB: mongodb.NewMongoDB(url),
	}
}

// Iter iterates over all machine documents and executes fn for each new
// iteration.
func (m *MongoDB) Iter(fn func(MachineDocument)) error {
	query := func(c *mgo.Collection) error {
		machinesWithIds := bson.M{
			"meta.instanceId": bson.M{"$exists": true, "$ne": ""},
		}

		machine := MachineDocument{}
		iter := c.Find(machinesWithIds).Batch(150).Iter()
		for iter.Next(&machine) {
			fn(machine)
		}

		return iter.Close()
	}

	return m.DB.Run("jMachines", query)
}

// AlwaysOn returns all alwaysOn Machines
func (m *MongoDB) AlwaysOn() ([]MachineDocument, error) {
	machines := make([]MachineDocument, 0)

	query := func(c *mgo.Collection) error {
		alwaysOn := bson.M{
			"meta.alwaysOn": true,
		}

		machine := MachineDocument{}
		iter := c.Find(alwaysOn).Batch(150).Iter()
		for iter.Next(&machine) {
			machines = append(machines, machine)
		}

		return iter.Close()
	}

	if err := m.DB.Run("jMachines", query); err != nil {
		return nil, err
	}

	return machines, nil
}

// Machines returns a list of machines for the given instanceIds
func (m *MongoDB) Machines(instanceIds ...string) ([]MachineDocument, error) {
	machines := make([]MachineDocument, 0)

	query := func(c *mgo.Collection) error {
		all := bson.M{
			"meta.instanceId": bson.M{"$in": instanceIds},
		}

		machine := MachineDocument{}
		iter := c.Find(all).Batch(150).Iter()
		for iter.Next(&machine) {
			machines = append(machines, machine)
		}

		return iter.Close()
	}

	if err := m.DB.Run("jMachines", query); err != nil {
		return nil, err
	}

	return machines, nil
}

// Accounts returns a list of accounts for the give objectIds in non hex form
func (m *MongoDB) Accounts(ids ...string) ([]models.Account, error) {
	b := make([]bson.ObjectId, len(ids))
	for i, id := range ids {
		b[i] = bson.ObjectIdHex(id)
	}

	accounts := make([]models.Account, 0)

	query := func(c *mgo.Collection) error {
		all := bson.M{
			"_id": bson.M{"$in": b},
		}

		account := models.Account{}
		iter := c.Find(all).Batch(150).Iter()
		for iter.Next(&account) {
			accounts = append(accounts, account)
		}

		return iter.Close()
	}

	if err := m.DB.Run("jAccounts", query); err != nil {
		return nil, err
	}

	return accounts, nil
}

// RemoveAlwaysOn removes the alwaysOn flag for the given usernames
func (m *MongoDB) RemoveAlwaysOn(usernames ...string) error {
	query := func(c *mgo.Collection) error {
		_, err := c.UpdateAll(
			bson.M{
				"credential": bson.M{"$in": usernames},
				"provider":   "koding",
			},
			bson.M{"$set": bson.M{"meta.alwaysOn": false}},
		)

		return err
	}

	return m.DB.Run("jMachines", query)
}
<|endoftext|>


<commit_before>package response

import (
	"errors"
	"net/http"
	"socialapi/config"
	"socialapi/workers/helper"

	"github.com/koding/bongo"
)

var ErrContentNotFound = errors.New("content not found")

// NewBadRequest is creating a new http response with predifined
// http response properties
func NewBadRequest(err error) (int, http.Header, interface{}, error) {
	if err == nil {
		err = errors.New("request is not valid")
	}

	// make sure errors are outputted
	helper.MustGetLogger().Error("Bad Request: %s", err)

	// do not expose errors to the client
	env := config.MustGet().Environment
	// do not expose errors to the client.
	if env != "dev" || env != "test" {
		err = genericError
	}

	return http.StatusBadRequest, nil, nil, BadRequest{err}
}

// NewAccessDenied sends access denied response back to client
//
// here not to leak info about the resource
// do send NotFound err
func NewAccessDenied(err error) (int, http.Header, interface{}, error) {
	helper.MustGetLogger().Error("Access Denied Err: %s", err.Error())
	return NewNotFound()
}

// HandleResultAndError wraps the function calls and get its reponse,
// assuming the second parameter as error checks it if it is null or not
// if err nor found, returns OK response
func HandleResultAndError(res interface{}, err error) (int, http.Header, interface{}, error) {
	if err == bongo.RecordNotFound {
		return NewNotFound()
	}

	if err != nil {
		return NewBadRequest(err)
	}

	return NewOK(res)
}

// HandleResultAndClientError is same as `HandleResultAndError`, but it
// returns the actual error to client as opposed to generic error.
func HandleResultAndClientError(res interface{}, err error) (int, http.Header, interface{}, error) {
	if err != nil {
		return http.StatusBadRequest, nil, nil, err
	}

	return NewOK(res)
}

// NewOK returns http StatusOK response
func NewOK(res interface{}) (int, http.Header, interface{}, error) {
	return http.StatusOK, nil, res, nil
}

// NewNotFound returns http StatusNotFound response
func NewNotFound() (int, http.Header, interface{}, error) {
	return http.StatusNotFound, nil, nil, NotFoundError{ErrContentNotFound}
}

// NewDeleted returns http StatusAccepted response
func NewDeleted() (int, http.Header, interface{}, error) {
	return http.StatusAccepted, nil, nil, nil
}

// NewDefaultOK returns http StatusOK response with `{status:true}` response
func NewDefaultOK() (int, http.Header, interface{}, error) {
	res := map[string]interface{}{
		"status": true,
	}

	return NewOK(res)
}
<commit_msg>wercker: fix social test error expose<commit_after>package response

import (
	"errors"
	"net/http"
	"socialapi/config"
	"socialapi/workers/helper"

	"github.com/koding/bongo"
)

var ErrContentNotFound = errors.New("content not found")

// NewBadRequest is creating a new http response with predifined
// http response properties
func NewBadRequest(err error) (int, http.Header, interface{}, error) {
	if err == nil {
		err = errors.New("request is not valid")
	}

	// make sure errors are outputted
	helper.MustGetLogger().Error("Bad Request: %s", err)

	// do not expose errors to the client
	env := config.MustGet().Environment
	// do not expose errors to the client.
	if env != "dev" && env != "test" {
		err = genericError
	}

	return http.StatusBadRequest, nil, nil, BadRequest{err}
}

// NewAccessDenied sends access denied response back to client
//
// here not to leak info about the resource
// do send NotFound err
func NewAccessDenied(err error) (int, http.Header, interface{}, error) {
	helper.MustGetLogger().Error("Access Denied Err: %s", err.Error())
	return NewNotFound()
}

// HandleResultAndError wraps the function calls and get its reponse,
// assuming the second parameter as error checks it if it is null or not
// if err nor found, returns OK response
func HandleResultAndError(res interface{}, err error) (int, http.Header, interface{}, error) {
	if err == bongo.RecordNotFound {
		return NewNotFound()
	}

	if err != nil {
		return NewBadRequest(err)
	}

	return NewOK(res)
}

// HandleResultAndClientError is same as `HandleResultAndError`, but it
// returns the actual error to client as opposed to generic error.
func HandleResultAndClientError(res interface{}, err error) (int, http.Header, interface{}, error) {
	if err != nil {
		return http.StatusBadRequest, nil, nil, err
	}

	return NewOK(res)
}

// NewOK returns http StatusOK response
func NewOK(res interface{}) (int, http.Header, interface{}, error) {
	return http.StatusOK, nil, res, nil
}

// NewNotFound returns http StatusNotFound response
func NewNotFound() (int, http.Header, interface{}, error) {
	return http.StatusNotFound, nil, nil, NotFoundError{ErrContentNotFound}
}

// NewDeleted returns http StatusAccepted response
func NewDeleted() (int, http.Header, interface{}, error) {
	return http.StatusAccepted, nil, nil, nil
}

// NewDefaultOK returns http StatusOK response with `{status:true}` response
func NewDefaultOK() (int, http.Header, interface{}, error) {
	res := map[string]interface{}{
		"status": true,
	}

	return NewOK(res)
}
<|endoftext|>


<commit_before>package buffalo

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

func TestCookies_Get(t *testing.T) {
	r := require.New(t)
	req := httptest.NewRequest("POST", "/", nil)
	req.Header.Set("Cookie", "name=Arthur Dent; answer=42")

	c := Cookies{req, nil}

	v, err := c.Get("name")
	r.NoError(err)
	r.Equal("Arthur Dent", v)

	v, err = c.Get("answer")
	r.NoError(err)
	r.Equal("42", v)

	_, err = c.Get("unknown")
	r.EqualError(err, http.ErrNoCookie.Error())
}

func TestCookies_Set(t *testing.T) {
	r := require.New(t)
	res := httptest.NewRecorder()

	c := Cookies{&http.Request{}, res}

	c.Set("name", "Rob Pike", time.Hour*24)

	h := res.Header().Get("Set-Cookie")
	r.Equal("name=Rob Pike; Max-Age=86400", h)
}

func TestCookies_SetWithExpirationTime(t *testing.T) {
	r := require.New(t)
	res := httptest.NewRecorder()

	c := Cookies{&http.Request{}, res}

	e := time.Date(2017, 7, 29, 19, 28, 45, 0, time.UTC)
	c.SetWithExpirationTime("name", "Rob Pike", e)

	h := res.Header().Get("Set-Cookie")
	r.Equal("name=Rob Pike; Expires=Sat, 29 Jul 2017 19:28:45 GMT", h)
}

func TestCookies_Delete(t *testing.T) {
	r := require.New(t)
	res := httptest.NewRecorder()

	c := Cookies{&http.Request{}, res}

	c.Delete("remove-me")

	h := res.Header().Get("Set-Cookie")
	r.Equal("remove-me=v; Expires=Thu, 01 Jan 1970 00:00:00 GMT", h)
}
<commit_msg>reverting pike's fix<commit_after>package buffalo

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

func TestCookies_Get(t *testing.T) {
	r := require.New(t)
	req := httptest.NewRequest("POST", "/", nil)
	req.Header.Set("Cookie", "name=Arthur Dent; answer=42")

	c := Cookies{req, nil}

	v, err := c.Get("name")
	r.NoError(err)
	r.Equal("Arthur Dent", v)

	v, err = c.Get("answer")
	r.NoError(err)
	r.Equal("42", v)

	_, err = c.Get("unknown")
	r.EqualError(err, http.ErrNoCookie.Error())
}

func TestCookies_Set(t *testing.T) {
	r := require.New(t)
	res := httptest.NewRecorder()

	c := Cookies{&http.Request{}, res}

	c.Set("name", "Rob Pike", time.Hour*24)

	h := res.Header().Get("Set-Cookie")
	r.Equal("name=\"Rob Pike\"; Max-Age=86400", h)
}

func TestCookies_SetWithExpirationTime(t *testing.T) {
	r := require.New(t)
	res := httptest.NewRecorder()

	c := Cookies{&http.Request{}, res}

	e := time.Date(2017, 7, 29, 19, 28, 45, 0, time.UTC)
	c.SetWithExpirationTime("name", "Rob Pike", e)

	h := res.Header().Get("Set-Cookie")
	r.Equal("name=\"Rob Pike\"; Expires=Sat, 29 Jul 2017 19:28:45 GMT", h)
}

func TestCookies_Delete(t *testing.T) {
	r := require.New(t)
	res := httptest.NewRecorder()

	c := Cookies{&http.Request{}, res}

	c.Delete("remove-me")

	h := res.Header().Get("Set-Cookie")
	r.Equal("remove-me=v; Expires=Thu, 01 Jan 1970 00:00:00 GMT", h)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Code from https://github.com/gogits/gogs/blob/v0.7.0/modules/avatar/avatar.go

package avatar

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/setting"
	"gopkg.in/macaron.v1"

	gocache "github.com/patrickmn/go-cache"
)

var gravatarSource string

func UpdateGravatarSource() {
	srcCfg := "//secure.gravatar.com/avatar/"

	gravatarSource = srcCfg
	if strings.HasPrefix(gravatarSource, "//") {
		gravatarSource = "http:" + gravatarSource
	} else if !strings.HasPrefix(gravatarSource, "http://") &&
		!strings.HasPrefix(gravatarSource, "https://") {
		gravatarSource = "http://" + gravatarSource
	}
}

// hash email to md5 string
// keep this func in order to make this package independent
func HashEmail(email string) string {
	// https://en.gravatar.com/site/implement/hash/
	email = strings.TrimSpace(email)
	email = strings.ToLower(email)

	h := md5.New()
	h.Write([]byte(email))
	return hex.EncodeToString(h.Sum(nil))
}

// Avatar represents the avatar object.
type Avatar struct {
	hash      string
	reqParams string
	data      *bytes.Buffer
	notFound  bool
	timestamp time.Time
}

func New(hash string) *Avatar {
	return &Avatar{
		hash: hash,
		reqParams: url.Values{
			"d":    {"retro"},
			"size": {"200"},
			"r":    {"pg"}}.Encode(),
	}
}

func (this *Avatar) Expired() bool {
	return time.Since(this.timestamp) > (time.Minute * 10)
}

func (this *Avatar) Encode(wr io.Writer) error {
	_, err := wr.Write(this.data.Bytes())
	return err
}

func (this *Avatar) Update() (err error) {
	select {
	case <-time.After(time.Second * 3):
		err = fmt.Errorf("get gravatar image %s timeout", this.hash)
	case err = <-thunder.GoFetch(gravatarSource+this.hash+"?"+this.reqParams, this):
	}
	return err
}

type CacheServer struct {
	notFound *Avatar
	cache    *gocache.Cache
}

func (this *CacheServer) mustInt(r *http.Request, defaultValue int, keys ...string) (v int) {
	for _, k := range keys {
		if _, err := fmt.Sscanf(r.FormValue(k), "%d", &v); err == nil {
			defaultValue = v
		}
	}
	return defaultValue
}

func (this *CacheServer) Handler(ctx *macaron.Context) {
	urlPath := ctx.Req.URL.Path
	hash := urlPath[strings.LastIndex(urlPath, "/")+1:]

	var avatar *Avatar

	if obj, exist := this.cache.Get(hash); exist {
		avatar = obj.(*Avatar)
	} else {
		avatar = New(hash)
	}

	if avatar.Expired() {
		if err := avatar.Update(); err != nil {
			log.Trace("avatar update error: %v", err)
			avatar = this.notFound
		}
	}

	if avatar.notFound {
		avatar = this.notFound
	} else {
		this.cache.Add(hash, avatar, gocache.DefaultExpiration)
	}

	ctx.Resp.Header().Add("Content-Type", "image/jpeg")

	if !setting.EnableGzip {
		ctx.Resp.Header().Add("Content-Length", strconv.Itoa(len(avatar.data.Bytes())))
	}

	ctx.Resp.Header().Add("Cache-Control", "private, max-age=3600")

	if err := avatar.Encode(ctx.Resp); err != nil {
		log.Warn("avatar encode error: %v", err)
		ctx.WriteHeader(500)
	}
}

func NewCacheServer() *CacheServer {
	UpdateGravatarSource()

	return &CacheServer{
		notFound: newNotFound(),
		cache:    gocache.New(time.Hour, time.Hour*2),
	}
}

func newNotFound() *Avatar {
	avatar := &Avatar{notFound: true}

	// load transparent png into buffer
	path := filepath.Join(setting.StaticRootPath, "img", "transparent.png")

	if data, err := ioutil.ReadFile(path); err != nil {
		log.Error(3, "Failed to read transparent.png, %v", path)
	} else {
		avatar.data = bytes.NewBuffer(data)
	}

	return avatar
}

// thunder downloader
var thunder = &Thunder{QueueSize: 10}

type Thunder struct {
	QueueSize int // download queue size
	q         chan *thunderTask
	once      sync.Once
}

func (t *Thunder) init() {
	if t.QueueSize < 1 {
		t.QueueSize = 1
	}
	t.q = make(chan *thunderTask, t.QueueSize)
	for i := 0; i < t.QueueSize; i++ {
		go func() {
			for {
				task := <-t.q
				task.Fetch()
			}
		}()
	}
}

func (t *Thunder) Fetch(url string, avatar *Avatar) error {
	t.once.Do(t.init)
	task := &thunderTask{
		Url:    url,
		Avatar: avatar,
	}
	task.Add(1)
	t.q <- task
	task.Wait()
	return task.err
}

func (t *Thunder) GoFetch(url string, avatar *Avatar) chan error {
	c := make(chan error)
	go func() {
		c <- t.Fetch(url, avatar)
	}()
	return c
}

// thunder download
type thunderTask struct {
	Url    string
	Avatar *Avatar
	sync.WaitGroup
	err error
}

func (this *thunderTask) Fetch() {
	this.err = this.fetch()
	this.Done()
}

var client *http.Client = &http.Client{
	Timeout:   time.Second * 2,
	Transport: &http.Transport{Proxy: http.ProxyFromEnvironment},
}

func (this *thunderTask) fetch() error {
	this.Avatar.timestamp = time.Now()

	log.Debug("avatar.fetch(fetch new avatar): %s", this.Url)
	req, _ := http.NewRequest("GET", this.Url, nil)
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/jpeg,image/png,*/*;q=0.8")
	req.Header.Set("Accept-Encoding", "deflate,sdch")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36")
	resp, err := client.Do(req)

	if err != nil {
		this.Avatar.notFound = true
		return fmt.Errorf("gravatar unreachable, %v", err)
	}

	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		this.Avatar.notFound = true
		return fmt.Errorf("status code: %d", resp.StatusCode)
	}

	this.Avatar.data = &bytes.Buffer{}
	writer := bufio.NewWriter(this.Avatar.data)

	if _, err = io.Copy(writer, resp.Body); err != nil {
		return err
	}

	return nil
}
<commit_msg>fix missing profile icon (#10469)<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Code from https://github.com/gogits/gogs/blob/v0.7.0/modules/avatar/avatar.go

package avatar

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/setting"
	"gopkg.in/macaron.v1"

	gocache "github.com/patrickmn/go-cache"
)

var gravatarSource string

func UpdateGravatarSource() {
	srcCfg := "//secure.gravatar.com/avatar/"

	gravatarSource = srcCfg
	if strings.HasPrefix(gravatarSource, "//") {
		gravatarSource = "http:" + gravatarSource
	} else if !strings.HasPrefix(gravatarSource, "http://") &&
		!strings.HasPrefix(gravatarSource, "https://") {
		gravatarSource = "http://" + gravatarSource
	}
}

// hash email to md5 string
// keep this func in order to make this package independent
func HashEmail(email string) string {
	// https://en.gravatar.com/site/implement/hash/
	email = strings.TrimSpace(email)
	email = strings.ToLower(email)

	h := md5.New()
	h.Write([]byte(email))
	return hex.EncodeToString(h.Sum(nil))
}

// Avatar represents the avatar object.
type Avatar struct {
	hash      string
	reqParams string
	data      *bytes.Buffer
	notFound  bool
	timestamp time.Time
}

func New(hash string) *Avatar {
	return &Avatar{
		hash: hash,
		reqParams: url.Values{
			"d":    {"retro"},
			"size": {"200"},
			"r":    {"pg"}}.Encode(),
	}
}

func (this *Avatar) Expired() bool {
	return time.Since(this.timestamp) > (time.Minute * 10)
}

func (this *Avatar) Encode(wr io.Writer) error {
	_, err := wr.Write(this.data.Bytes())
	return err
}

func (this *Avatar) Update() (err error) {
	select {
	case <-time.After(time.Second * 3):
		err = fmt.Errorf("get gravatar image %s timeout", this.hash)
	case err = <-thunder.GoFetch(gravatarSource+this.hash+"?"+this.reqParams, this):
	}
	return err
}

type CacheServer struct {
	notFound *Avatar
	cache    *gocache.Cache
}

func (this *CacheServer) mustInt(r *http.Request, defaultValue int, keys ...string) (v int) {
	for _, k := range keys {
		if _, err := fmt.Sscanf(r.FormValue(k), "%d", &v); err == nil {
			defaultValue = v
		}
	}
	return defaultValue
}

func (this *CacheServer) Handler(ctx *macaron.Context) {
	urlPath := ctx.Req.URL.Path
	hash := urlPath[strings.LastIndex(urlPath, "/")+1:]

	var avatar *Avatar

	if obj, exist := this.cache.Get(hash); exist {
		avatar = obj.(*Avatar)
	} else {
		avatar = New(hash)
	}

	if avatar.Expired() {
		if err := avatar.Update(); err != nil {
			log.Trace("avatar update error: %v", err)
			avatar = this.notFound
		}
	}

	if avatar.notFound {
		avatar = this.notFound
	} else {
		this.cache.Add(hash, avatar, gocache.DefaultExpiration)
	}

	ctx.Resp.Header().Add("Content-Type", "image/jpeg")

	if !setting.EnableGzip {
		ctx.Resp.Header().Add("Content-Length", strconv.Itoa(len(avatar.data.Bytes())))
	}

	ctx.Resp.Header().Add("Cache-Control", "private, max-age=3600")

	if err := avatar.Encode(ctx.Resp); err != nil {
		log.Warn("avatar encode error: %v", err)
		ctx.WriteHeader(500)
	}
}

func NewCacheServer() *CacheServer {
	UpdateGravatarSource()

	return &CacheServer{
		notFound: newNotFound(),
		cache:    gocache.New(time.Hour, time.Hour*2),
	}
}

func newNotFound() *Avatar {
	avatar := &Avatar{notFound: true}

	// load user_profile png into buffer
	path := filepath.Join(setting.StaticRootPath, "img", "user_profile.png")

	if data, err := ioutil.ReadFile(path); err != nil {
		log.Error(3, "Failed to read user_profile.png, %v", path)
	} else {
		avatar.data = bytes.NewBuffer(data)
	}

	return avatar
}

// thunder downloader
var thunder = &Thunder{QueueSize: 10}

type Thunder struct {
	QueueSize int // download queue size
	q         chan *thunderTask
	once      sync.Once
}

func (t *Thunder) init() {
	if t.QueueSize < 1 {
		t.QueueSize = 1
	}
	t.q = make(chan *thunderTask, t.QueueSize)
	for i := 0; i < t.QueueSize; i++ {
		go func() {
			for {
				task := <-t.q
				task.Fetch()
			}
		}()
	}
}

func (t *Thunder) Fetch(url string, avatar *Avatar) error {
	t.once.Do(t.init)
	task := &thunderTask{
		Url:    url,
		Avatar: avatar,
	}
	task.Add(1)
	t.q <- task
	task.Wait()
	return task.err
}

func (t *Thunder) GoFetch(url string, avatar *Avatar) chan error {
	c := make(chan error)
	go func() {
		c <- t.Fetch(url, avatar)
	}()
	return c
}

// thunder download
type thunderTask struct {
	Url    string
	Avatar *Avatar
	sync.WaitGroup
	err error
}

func (this *thunderTask) Fetch() {
	this.err = this.fetch()
	this.Done()
}

var client *http.Client = &http.Client{
	Timeout:   time.Second * 2,
	Transport: &http.Transport{Proxy: http.ProxyFromEnvironment},
}

func (this *thunderTask) fetch() error {
	this.Avatar.timestamp = time.Now()

	log.Debug("avatar.fetch(fetch new avatar): %s", this.Url)
	req, _ := http.NewRequest("GET", this.Url, nil)
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/jpeg,image/png,*/*;q=0.8")
	req.Header.Set("Accept-Encoding", "deflate,sdch")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36")
	resp, err := client.Do(req)

	if err != nil {
		this.Avatar.notFound = true
		return fmt.Errorf("gravatar unreachable, %v", err)
	}

	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		this.Avatar.notFound = true
		return fmt.Errorf("status code: %d", resp.StatusCode)
	}

	this.Avatar.data = &bytes.Buffer{}
	writer := bufio.NewWriter(this.Avatar.data)

	if _, err = io.Copy(writer, resp.Body); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Cisco Systems, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apicapi

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"sort"

	"github.com/sirupsen/logrus"
)

func (conn *ApicConnection) apicBodyAttrCmp(class string,
	bodyc *ApicObjectBody, bodyd *ApicObjectBody) bool {
	meta, ok := metadata[class]
	if !ok {
		conn.log.Warning("No metadata for class ", class)
		return true
	}
	if bodyc.Attributes == nil {
		bodyc.Attributes = make(map[string]interface{})
	}
	if bodyd.Attributes == nil {
		bodyd.Attributes = make(map[string]interface{})
	}
	for p, def := range meta.attributes {
		ac, ok := bodyc.Attributes[p]
		if !ok {
			ac = def
		}
		ad, ok := bodyd.Attributes[p]
		if !ok {
			ad = def
		}

		if ac != ad {
			return false
		}
	}
	if class != "tagAnnotation" && class != "tagInst" {
		annotc, ok := bodyc.Attributes["annotation"]
		if !ok {
			annotc = ""
		}
		annotd, ok := bodyd.Attributes["annotation"]
		if !ok {
			annotd = ""
		}
		if annotc != annotd {
			return false
		}
	}

	return true
}

func (conn *ApicConnection) apicCntCmp(current ApicObject,
	desired ApicObject) bool {
	for classc, bodyc := range current {
		for classd, bodyd := range desired {
			if classc != classd {
				conn.log.Warning("Invalid comparison ", classc, " != ", classd)
				return false
			}

			return conn.apicBodyAttrCmp(classc, bodyc, bodyd)
		}
	}
	return true
}

func (conn *ApicConnection) apicObjCmp(current ApicObject,
	desired ApicObject) (update bool, deletes []string) {

	for classc, bodyc := range current {
		for classd, bodyd := range desired {
			if classc != classd {
				conn.log.Warning("Invalid comparison ", classc, " != ", classd)
				return
			}

			if !conn.apicBodyAttrCmp(classc, bodyc, bodyd) {
				update = true
			}

			i := 0
			j := 0
			for i < len(bodyc.Children) && j < len(bodyd.Children) {
				cmp := cmpApicObject(bodyc.Children[i], bodyd.Children[j])
				if cmp < 0 {
					deletes = append(deletes, bodyc.Children[i].GetDn())
					i++
				} else if cmp > 0 {
					update = true
					j++
				} else {
					cu, cd := conn.apicObjCmp(bodyc.Children[i],
						bodyd.Children[j])
					if cu {
						update = true
					}
					deletes = append(deletes, cd...)

					i++
					j++
				}
			}
			for i < len(bodyc.Children) {
				deletes = append(deletes, bodyc.Children[i].GetDn())
				i++
			}
			if j < len(bodyd.Children) {
				update = true
			}
		}
	}
	//if update {
	//	conn.log.WithFields(logrus.Fields{
	//		"current": current,
	//		"desired": desired,
	//	}).Debug("Update required")
	//}
	return
}

func (conn *ApicConnection) diffApicState(currentState ApicSlice,
	desiredState ApicSlice) (updates ApicSlice, deletes []string) {

	i := 0
	j := 0

	for i < len(currentState) && j < len(desiredState) {
		cmp := cmpApicObject(currentState[i], desiredState[j])
		if cmp < 0 {
			deletes = append(deletes, currentState[i].GetDn())
			i++
		} else if cmp > 0 {
			updates = append(updates, desiredState[j])
			j++
		} else {
			if conn.containerDns[currentState[i].GetDn()] {
				if !conn.apicCntCmp(currentState[i], desiredState[j]) {
					updates = append(updates, desiredState[j])
				}
			} else {
				cu, cd := conn.apicObjCmp(currentState[i], desiredState[j])
				if cu {
					updates = append(updates, desiredState[j])
				}
				deletes = append(deletes, cd...)
			}

			i++
			j++
		}
	}
	// extra old objects
	for i < len(currentState) {
		deletes = append(deletes, currentState[i].GetDn())
		i++
	}
	// extra new objects
	for j < len(desiredState) {
		updates = append(updates, desiredState[j])
		j++
	}

	return
}

func (conn *ApicConnection) applyDiff(updates ApicSlice, deletes []string,
	context string) {
	sort.Sort(updates)
	sort.Strings(deletes)

	for _, delete := range deletes {
		conn.log.WithFields(logrus.Fields{"mod": "APICAPI", "DN": delete, "context": context}).
			Debug("Applying APIC object delete")
		conn.queueDn(delete)
	}
	for _, update := range updates {
		dn := update.GetDn()
		conn.log.WithFields(logrus.Fields{"mod": "APICAPI", "DN": dn, "context": context}).
			Debug("Applying APIC object update")
		conn.queueDn(dn)
	}
}

func getTagFromKey(prefix string, key string) string {
	hash := sha256.Sum256([]byte(key))
	return fmt.Sprintf("%s-%s", prefix, hex.EncodeToString(hash[:16]))
}

func PrepareApicSlice(objects ApicSlice, prefix string, key string) ApicSlice {
	return prepareApicSliceTag(objects, getTagFromKey(prefix, key))
}

func prepareApicSliceTag(objects ApicSlice, tag string) ApicSlice {
	sort.Sort(objects)
	for _, obj := range objects {
		for class, body := range obj {
			if class != "tagInst" && class != "tagAnnotation" {
				obj.SetTag(tag)
				if body.Attributes == nil {
					body.Attributes = make(map[string]interface{})
				}
				body.Attributes["annotation"] =
					aciContainersOwnerAnnotation
			}
			prepareApicSliceTag(body.Children, tag)

			if md, ok := metadata[class]; ok {
				if md.normalizer != nil {
					md.normalizer(body)
				}
			}
			break
		}
	}
	return objects
}

func prepareApicCache(parentDn string, obj ApicObject) {
	for _, body := range obj {
		if body.Attributes == nil {
			body.Attributes = make(map[string]interface{})
		}
		dn := obj.BuildDn(parentDn)
		body.Attributes["dn"] = dn

		for _, c := range body.Children {
			prepareApicCache(dn, c)
		}
		sort.Sort(body.Children)
		break
	}
}

func (conn *ApicConnection) fullSync() {
	conn.log.Info("Starting APIC full sync")
	var updates ApicSlice
	var deletes []string

	conn.indexMutex.Lock()
	for tag, current := range conn.cachedState {
		sort.Sort(current)
		key := conn.keyHashes[tag]
		u, d := conn.diffApicState(current, conn.desiredState[key])
		updates = append(updates, u...)
		deletes = append(deletes, d...)
	}

	for key, desired := range conn.desiredState {
		tag := getTagFromKey(conn.prefix, key)
		if _, ok := conn.cachedState[tag]; !ok {
			// entire key not present in current state
			a, _ := conn.diffApicState(nil, desired)
			updates = append(updates, a...)
		}
	}

	conn.syncEnabled = true
	conn.indexMutex.Unlock()

	conn.applyDiff(updates, deletes, "sync")
	conn.log.WithFields(logrus.Fields{
		"mod":     "APICAPI",
		"updates": len(updates),
		"deletes": len(deletes),
	}).Info("APIC full sync completed")
}

func (conn *ApicConnection) checkDeletes(oldState map[string]map[string]bool) {
	conn.indexMutex.Lock()
	for dn, ids := range oldState {
		_, found := conn.cacheDnSubIds[dn]
		if !found {
			for id := range ids {
				value, ok := conn.subscriptions.ids[id]
				if !ok {
					continue
				}
				sub, ok := conn.subscriptions.subs[value]
				if !ok {
					continue
				}
				if sub.deleteHook != nil {
					sub.deleteHook(dn)
				}
			}
		}
	}
	conn.indexMutex.Unlock()
}

func (conn *ApicConnection) updateDnIndex(objects ApicSlice) {
	for _, obj := range objects {
		conn.desiredStateDn[obj.GetDn()] = obj
		for _, body := range obj {
			conn.updateDnIndex(body.Children)
		}
	}
}

func (conn *ApicConnection) removeFromDnIndex(dn string) {
	if obj, ok := conn.desiredStateDn[dn]; ok {
		delete(conn.desiredStateDn, dn)

		for _, body := range obj {
			for _, child := range body.Children {
				conn.removeFromDnIndex(child.GetDn())
			}
		}
	}
}

func (conn *ApicConnection) doWriteApicObjects(key string, objects ApicSlice,
	container bool) {
	tag := getTagFromKey(conn.prefix, key)
	prepareApicSliceTag(objects, tag)

	conn.indexMutex.Lock()
	updates, deletes := conn.diffApicState(conn.desiredState[key], objects)

	conn.updateDnIndex(objects)
	for _, del := range deletes {
		conn.removeFromDnIndex(del)
		if container {
			delete(conn.containerDns, del)
		}
	}
	for _, update := range updates {
		dn := update.GetDn()
		if container {
			conn.containerDns[dn] = true
		}
	}

	if objects == nil {
		delete(conn.desiredState, key)
		delete(conn.keyHashes, tag)
	} else {
		conn.desiredState[key] = objects
		conn.keyHashes[tag] = key
	}

	if conn.syncEnabled {
		conn.indexMutex.Unlock()
		conn.applyDiff(updates, deletes, "write")
	} else {
		conn.indexMutex.Unlock()
	}
}

func (conn *ApicConnection) ClearApicContainer(key string) {
	conn.WriteApicContainer(key, nil)
}

func (conn *ApicConnection) WriteApicContainer(key string, objects ApicSlice) {
	conn.doWriteApicObjects(key, objects, true)
}

func (conn *ApicConnection) ClearApicObjects(key string) {
	conn.WriteApicObjects(key, nil)
}

func (conn *ApicConnection) WriteApicObjects(key string, objects ApicSlice) {
	conn.doWriteApicObjects(key, objects, false)
}

func (conn *ApicConnection) reconcileApicObject(aci ApicObject) {
	conn.indexMutex.Lock()
	if !conn.syncEnabled {
		conn.indexMutex.Unlock()
		return
	}

	dn := aci.GetDn()

	var updates ApicSlice
	var deletes []string

	if eobj, ok := conn.desiredStateDn[dn]; ok {
		if conn.containerDns[dn] {
			if !conn.apicCntCmp(aci, eobj) {
				updates = ApicSlice{eobj}
				conn.log.WithFields(logrus.Fields{
					"mod":      "APICAPI",
					"DN":       dn,
					"expected": eobj,
					"actual":   aci,
				}).Warning("Unexpected ACI container alteration")
			}
		} else {
			update, odels := conn.apicObjCmp(aci, eobj)
			if update {
				updates = ApicSlice{eobj}
			}
			deletes = append(deletes, odels...)

			if update || len(odels) != 0 {
				conn.log.WithFields(logrus.Fields{
					"mod":      "APICAPI",
					"DN":       dn,
					"expected": eobj,
					"actual":   aci,
				}).Warning("Unexpected ACI object alteration")
			}
		}
	} else {
		tag := aci.GetTag()
		if conn.isSyncTag(tag) {
			conn.log.WithFields(logrus.Fields{"mod": "APICAPI", "DN": dn}).
				Warning("Deleting unexpected ACI object")
			deletes = append(deletes, dn)
		}
	}

	conn.indexMutex.Unlock()

	conn.applyDiff(updates, deletes, "reconcile "+dn)
}
<commit_msg>Explicitly remove vnsRedirectDest from svcRedirectPol's list of children<commit_after>// Copyright 2017 Cisco Systems, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apicapi

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"sort"
	"strings"

	"github.com/sirupsen/logrus"
)

func (conn *ApicConnection) apicBodyAttrCmp(class string,
	bodyc *ApicObjectBody, bodyd *ApicObjectBody) bool {
	meta, ok := metadata[class]
	if !ok {
		conn.log.Warning("No metadata for class ", class)
		return true
	}
	if bodyc.Attributes == nil {
		bodyc.Attributes = make(map[string]interface{})
	}
	if bodyd.Attributes == nil {
		bodyd.Attributes = make(map[string]interface{})
	}
	for p, def := range meta.attributes {
		ac, ok := bodyc.Attributes[p]
		if !ok {
			ac = def
		}
		ad, ok := bodyd.Attributes[p]
		if !ok {
			ad = def
		}

		if ac != ad {
			return false
		}
	}
	if class != "tagAnnotation" && class != "tagInst" {
		annotc, ok := bodyc.Attributes["annotation"]
		if !ok {
			annotc = ""
		}
		annotd, ok := bodyd.Attributes["annotation"]
		if !ok {
			annotd = ""
		}
		if annotc != annotd {
			return false
		}
	}

	return true
}

func (conn *ApicConnection) apicCntCmp(current ApicObject,
	desired ApicObject) bool {
	for classc, bodyc := range current {
		for classd, bodyd := range desired {
			if classc != classd {
				conn.log.Warning("Invalid comparison ", classc, " != ", classd)
				return false
			}

			return conn.apicBodyAttrCmp(classc, bodyc, bodyd)
		}
	}
	return true
}

func (conn *ApicConnection) apicObjCmp(current ApicObject,
	desired ApicObject) (update bool, deletes []string) {

	for classc, bodyc := range current {
		for classd, bodyd := range desired {
			if classc != classd {
				conn.log.Warning("Invalid comparison ", classc, " != ", classd)
				return
			}

			if !conn.apicBodyAttrCmp(classc, bodyc, bodyd) {
				update = true
			}

			i := 0
			j := 0
			for i < len(bodyc.Children) && j < len(bodyd.Children) {
				cmp := cmpApicObject(bodyc.Children[i], bodyd.Children[j])
				if cmp < 0 {
					deletes = append(deletes, bodyc.Children[i].GetDn())
					i++
				} else if cmp > 0 {
					update = true
					j++
				} else {
					cu, cd := conn.apicObjCmp(bodyc.Children[i],
						bodyd.Children[j])
					if cu {
						update = true
					}
					deletes = append(deletes, cd...)

					i++
					j++
				}
			}
			for i < len(bodyc.Children) {
				deletes = append(deletes, bodyc.Children[i].GetDn())
				i++
			}
			if j < len(bodyd.Children) {
				update = true
			}
		}
	}
	//if update {
	//	conn.log.WithFields(logrus.Fields{
	//		"current": current,
	//		"desired": desired,
	//	}).Debug("Update required")
	//}
	return
}

func (conn *ApicConnection) diffApicState(currentState ApicSlice,
	desiredState ApicSlice) (updates ApicSlice, deletes []string) {

	i := 0
	j := 0

	for i < len(currentState) && j < len(desiredState) {
		cmp := cmpApicObject(currentState[i], desiredState[j])
		if cmp < 0 {
			deletes = append(deletes, currentState[i].GetDn())
			i++
		} else if cmp > 0 {
			updates = append(updates, desiredState[j])
			j++
		} else {
			if conn.containerDns[currentState[i].GetDn()] {
				if !conn.apicCntCmp(currentState[i], desiredState[j]) {
					updates = append(updates, desiredState[j])
				}
			} else {
				cu, cd := conn.apicObjCmp(currentState[i], desiredState[j])
				if cu {
					updates = append(updates, desiredState[j])
				}
				deletes = append(deletes, cd...)
			}

			i++
			j++
		}
	}
	// extra old objects
	for i < len(currentState) {
		deletes = append(deletes, currentState[i].GetDn())
		i++
	}
	// extra new objects
	for j < len(desiredState) {
		updates = append(updates, desiredState[j])
		j++
	}

	conn.log.Debug("Apic object updates are :", updates)
	conn.log.Debug("Apic object deletes are :", deletes)

	return
}

func (conn *ApicConnection) applyDiff(updates ApicSlice, deletes []string,
	context string) {
	sort.Sort(updates)
	sort.Strings(deletes)

	for _, delete := range deletes {
		conn.log.WithFields(logrus.Fields{"mod": "APICAPI", "DN": delete, "context": context}).
			Debug("Applying APIC object delete")
		conn.queueDn(delete)
	}
	for _, update := range updates {
		dn := update.GetDn()
		conn.log.WithFields(logrus.Fields{"mod": "APICAPI", "DN": dn, "context": context}).
			Debug("Applying APIC object update")
		conn.queueDn(dn)
	}
}

func getTagFromKey(prefix string, key string) string {
	hash := sha256.Sum256([]byte(key))
	return fmt.Sprintf("%s-%s", prefix, hex.EncodeToString(hash[:16]))
}

func PrepareApicSlice(objects ApicSlice, prefix string, key string) ApicSlice {
	return prepareApicSliceTag(objects, getTagFromKey(prefix, key))
}

func prepareApicSliceTag(objects ApicSlice, tag string) ApicSlice {
	sort.Sort(objects)
	for _, obj := range objects {
		for class, body := range obj {
			if class != "tagInst" && class != "tagAnnotation" {
				obj.SetTag(tag)
				if body.Attributes == nil {
					body.Attributes = make(map[string]interface{})
				}
				body.Attributes["annotation"] =
					aciContainersOwnerAnnotation
			}
			prepareApicSliceTag(body.Children, tag)

			if md, ok := metadata[class]; ok {
				if md.normalizer != nil {
					md.normalizer(body)
				}
			}
			break
		}
	}
	return objects
}

func prepareApicCache(parentDn string, obj ApicObject) {
	for _, body := range obj {
		if body.Attributes == nil {
			body.Attributes = make(map[string]interface{})
		}
		dn := obj.BuildDn(parentDn)
		body.Attributes["dn"] = dn

		for _, c := range body.Children {
			prepareApicCache(dn, c)
		}
		sort.Sort(body.Children)
		break
	}
}

func (conn *ApicConnection) fullSync() {
	conn.log.Info("Starting APIC full sync")
	var updates ApicSlice
	var deletes []string

	conn.indexMutex.Lock()
	for tag, current := range conn.cachedState {
		sort.Sort(current)
		key := conn.keyHashes[tag]
		u, d := conn.diffApicState(current, conn.desiredState[key])
		updates = append(updates, u...)
		deletes = append(deletes, d...)
	}

	for key, desired := range conn.desiredState {
		tag := getTagFromKey(conn.prefix, key)
		if _, ok := conn.cachedState[tag]; !ok {
			// entire key not present in current state
			a, _ := conn.diffApicState(nil, desired)
			updates = append(updates, a...)
		}
	}

	conn.syncEnabled = true
	conn.indexMutex.Unlock()

	conn.applyDiff(updates, deletes, "sync")
	conn.log.WithFields(logrus.Fields{
		"mod":     "APICAPI",
		"updates": len(updates),
		"deletes": len(deletes),
	}).Info("APIC full sync completed")
}

func (conn *ApicConnection) checkDeletes(oldState map[string]map[string]bool) {
	conn.indexMutex.Lock()
	for dn, ids := range oldState {
		_, found := conn.cacheDnSubIds[dn]
		if !found {
			for id := range ids {
				value, ok := conn.subscriptions.ids[id]
				if !ok {
					continue
				}
				sub, ok := conn.subscriptions.subs[value]
				if !ok {
					continue
				}
				if sub.deleteHook != nil {
					sub.deleteHook(dn)
				}
			}
		}
	}
	conn.indexMutex.Unlock()
}

func (conn *ApicConnection) updateDnIndex(objects ApicSlice) {
	for _, obj := range objects {
		conn.desiredStateDn[obj.GetDn()] = obj
		for _, body := range obj {
			conn.updateDnIndex(body.Children)
		}
	}
}

func (conn *ApicConnection) removeFromDnIndex(dn string) {
	if obj, ok := conn.desiredStateDn[dn]; ok {
		delete(conn.desiredStateDn, dn)

		for _, body := range obj {
			for _, child := range body.Children {
				conn.removeFromDnIndex(child.GetDn())
				conn.log.Debug("Removing child dn :", child.GetDn())
			}
		}
	}
}

func (conn *ApicConnection) doWriteApicObjects(key string, objects ApicSlice,
	container bool) {

	tag := getTagFromKey(conn.prefix, key)
	prepareApicSliceTag(objects, tag)

	conn.indexMutex.Lock()
	updates, deletes := conn.diffApicState(conn.desiredState[key], objects)

	// temp cache to store all the "uni/tn-common/svcCont/svcRedirectPol-kube_svc_default_test-master"
	// found in deletes
	var temp_deletes []string
	for _, delete := range deletes {
		if strings.Contains(delete, "svcRedirectPol") {
			temp_deletes = append(temp_deletes, delete)
		}
	}
	for _, temp_del := range temp_deletes {
		vns_svc_redirect_pol_obj, ok := conn.desiredStateDn[temp_del]
		if !ok {
			conn.log.Error("no svc_obj found in desiredStateDn cache")
			return
		}
		// Explicitly remove vnsRedirectDest from svcRedirectPol's list of children
		for _, body := range vns_svc_redirect_pol_obj {
			for _, child := range body.Children {
				for class := range child {
					if class == "vnsRedirectDest" {
						deletes = append(deletes, child.GetDn())
					}
				}
			}
		}
	}
	conn.log.Debug("Updated apic object deletes list is :", deletes)

	conn.updateDnIndex(objects)
	for _, del := range deletes {
		conn.removeFromDnIndex(del)
		if container {
			delete(conn.containerDns, del)
		}
	}
	for _, update := range updates {
		dn := update.GetDn()
		if container {
			conn.containerDns[dn] = true
		}
	}

	if objects == nil {
		delete(conn.desiredState, key)
		delete(conn.keyHashes, tag)
	} else {
		conn.desiredState[key] = objects
		conn.keyHashes[tag] = key
	}

	if conn.syncEnabled {
		conn.indexMutex.Unlock()
		conn.applyDiff(updates, deletes, "write")
	} else {
		conn.indexMutex.Unlock()
	}
}

func (conn *ApicConnection) ClearApicContainer(key string) {
	conn.WriteApicContainer(key, nil)
}

func (conn *ApicConnection) WriteApicContainer(key string, objects ApicSlice) {
	conn.doWriteApicObjects(key, objects, true)
}

func (conn *ApicConnection) ClearApicObjects(key string) {
	conn.WriteApicObjects(key, nil)
}

func (conn *ApicConnection) WriteApicObjects(key string, objects ApicSlice) {
	conn.doWriteApicObjects(key, objects, false)
}

func (conn *ApicConnection) reconcileApicObject(aci ApicObject) {
	conn.indexMutex.Lock()
	if !conn.syncEnabled {
		conn.indexMutex.Unlock()
		return
	}

	dn := aci.GetDn()

	var updates ApicSlice
	var deletes []string

	if eobj, ok := conn.desiredStateDn[dn]; ok {
		if conn.containerDns[dn] {
			if !conn.apicCntCmp(aci, eobj) {
				updates = ApicSlice{eobj}
				conn.log.WithFields(logrus.Fields{
					"mod":      "APICAPI",
					"DN":       dn,
					"expected": eobj,
					"actual":   aci,
				}).Warning("Unexpected ACI container alteration")
			}
		} else {
			update, odels := conn.apicObjCmp(aci, eobj)
			if update {
				updates = ApicSlice{eobj}
			}
			deletes = append(deletes, odels...)

			if update || len(odels) != 0 {
				conn.log.WithFields(logrus.Fields{
					"mod":      "APICAPI",
					"DN":       dn,
					"expected": eobj,
					"actual":   aci,
				}).Warning("Unexpected ACI object alteration")
			}
		}
	} else {
		tag := aci.GetTag()
		if conn.isSyncTag(tag) {
			conn.log.WithFields(logrus.Fields{"mod": "APICAPI", "DN": dn}).
				Warning("Deleting unexpected ACI object")
			deletes = append(deletes, dn)
		}
	}

	conn.indexMutex.Unlock()

	conn.applyDiff(updates, deletes, "reconcile "+dn)
}
<|endoftext|>


<commit_before>package local

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/docker/infrakit/pkg/template"
	"github.com/docker/infrakit/pkg/util/exec"
	"github.com/spf13/cobra"
)

const (
	// none is used to determine if the user has set the bool flag value. this allows the
	// use of pipe to a prompt like {{ $foo = flag "flag" "bool" "message" | prompt "foo?" "bool" }}
	none = -1
)

// Context is the context for the running module
type Context struct {
	cmd    *cobra.Command
	src    string
	input  io.Reader
	exec   bool
	run    func(string) error
	script string
}

// name, type, description of the flag, and a default value, which can be nil
// the returned value if the nil value
func (c *Context) defineFlag(name, ftype, desc string, def interface{}) (interface{}, error) {
	switch ftype {

	case "string":
		defaultValue := ""
		if def != nil {
			if v, ok := def.(string); ok {
				defaultValue = v
			} else {
				return nil, fmt.Errorf("default value not a string: %s", name)
			}
		}
		c.cmd.Flags().String(name, defaultValue, desc)
		return defaultValue, nil

	case "int":
		defaultValue := 0 // TODO - encode a nil with a special value?
		if def != nil {
			if v, ok := def.(int); ok {
				defaultValue = v
			} else {
				return nil, fmt.Errorf("default value not an int: %s", name)
			}
		}
		c.cmd.Flags().Int(name, defaultValue, desc)
		return defaultValue, nil

	case "float":
		defaultValue := 0.
		if def != nil {
			if v, ok := def.(float64); ok {
				defaultValue = v
			} else {
				return nil, fmt.Errorf("default value not a float64: %s", name)
			}
		}
		c.cmd.Flags().Float64(name, defaultValue, desc)
		return defaultValue, nil

	case "bool":
		// bool is special in that we want to handle the case of nil --> the flag is not specified
		// so that we can pipe to a prompt if necessary.  pflag does not have the notion of unset
		// flags, so we'd have to hack around it by introducing a string flag if the default is not
		// specified in the template function to define the flag.
		if def != nil {
			// When the default is specified, we cannot use a pipe to prompt.
			// So here just create a bool flag
			defaultValue := false
			switch v := def.(type) {
			case bool:
				defaultValue = v
			case string:
				b, err := strconv.ParseBool(v)
				if err != nil {
					return nil, err
				}
				defaultValue = b
			}
			c.cmd.Flags().Bool(name, defaultValue, desc)
			return defaultValue, nil
		}
		// At definition time, there is no default value, so we use string
		// to model three states: true, false, none
		c.cmd.Flags().Int(name, none, desc)
		return none, nil
	}
	return nil, fmt.Errorf("unknown type %v", ftype)
}

// name, type, description, and default value that can be nil
func (c *Context) getFromFlag(name, ftype, desc string, def interface{}) (interface{}, error) {

	switch ftype {

	case "string":
		return c.cmd.Flags().GetString(name)

	case "int":
		return c.cmd.Flags().GetInt(name)

	case "float":
		return c.cmd.Flags().GetFloat64(name)

	case "bool":
		if def == nil {
			// If default v is not specified, then we assume the flag was defined
			// using a string to handle the none case
			v, err := c.cmd.Flags().GetInt(name)
			if err != nil {
				return none, err
			}
			if v == none {
				return none, nil //
			}
			return v > 0, nil
		}
		return c.cmd.Flags().GetBool(name)
	}

	return nil, nil
}

// returns true if the value v is missing of the type t
func missing(t string, v interface{}) bool {
	if v == nil {
		return true
	}
	switch t {
	case "string":
		return v.(string) == ""
	case "int":
		return v.(int) == 0
	case "float":
		return v.(float64) == 0.
	case "bool":
		return v == none
	}
	return true
}

func (c *Context) prompt(prompt, ftype string) (interface{}, error) {
	input := bufio.NewReader(c.input)
	fmt.Fprintf(os.Stderr, "%s ", prompt)
	text, _ := input.ReadString('\n')
	text = strings.Trim(text, " \t\n")
	switch ftype {
	case "string":
		return text, nil
	case "int":
		return strconv.Atoi(text)
	case "float":
		return strconv.ParseFloat(text, 64)
	case "bool":
		if b, err := strconv.ParseBool(text); err == nil {
			return b, nil
		}
		switch text {
		case "y", "Y", "yes", "ok", "OK":
			return true, nil
		}
		if v, err := strconv.Atoi(text); err == nil {
			return v > 0, nil
		}
		return nil, fmt.Errorf("cannot parse input for boolean: %v", text)
	}
	return nil, nil
}

// Funcs returns the template functions
func (c *Context) Funcs() []template.Function {
	return []template.Function{
		{
			Name: "flag",
			Func: func(n, ftype, desc string, optional ...interface{}) (interface{}, error) {
				if ftype == "" {
					return nil, fmt.Errorf("missing type for variable %v", n)
				}
				var defaultValue interface{}
				if len(optional) > 0 {
					defaultValue = optional[0]
				}
				if c.exec {
					return c.getFromFlag(n, ftype, desc, defaultValue)
				}

				// Defining a flag never returns a printable value that can mess up the template rendering
				return c.defineFlag(n, ftype, desc, defaultValue)
			},
		},
		{
			Name: "prompt",
			Func: func(prompt, ftype string, optional ...interface{}) (interface{}, error) {

				if ftype == "" {
					return nil, fmt.Errorf("missing type for variable prompted")
				}
				if !c.exec {
					return "", nil
				}
				if len(optional) > 0 && !missing(ftype, optional[0]) {
					return optional[0], nil
				}
				return c.prompt(prompt, ftype)
			},
		},
	}
}

// loadBackend determines the backend to use for executing the rendered template text (e.g. run in shell).
// During this phase, the template delimiters are changed to =% %= so put this in the comment {{/* */}}
func (c *Context) loadBackend() error {
	t, err := template.NewTemplate(c.src, template.Options{
		DelimLeft:  "=%",
		DelimRight: "%=",
	})
	if err != nil {
		return err
	}
	t.AddFunc("print",
		func() string {
			c.run = func(script string) error {
				fmt.Println(script)
				return nil
			}
			return ""
		})
	t.AddFunc("sh",
		func(opts ...string) string {
			c.run = func(script string) error {

				cmd := strings.Join(append([]string{"/bin/sh"}, opts...), " ")
				log.Debug("sh", "cmd", cmd)

				return exec.Command(cmd).
					InheritEnvs(true).StartWithStreams(

					exec.Do(exec.SendInput(
						func(stdin io.WriteCloser) error {
							_, err := stdin.Write([]byte(script))
							return err
						})).Then(
						exec.RedirectStdout(os.Stdout)).Then(
						exec.RedirectStderr(os.Stderr),
					).Done(),
				)
			}
			return ""
		})

	_, err = t.Render(c)
	return err
}

// buildFlags from parsing the body which is a template
func (c *Context) buildFlags() error {
	t, err := template.NewTemplate(c.src, template.Options{})
	if err != nil {
		return err
	}
	_, err = t.Render(c)
	return err
}

func (c *Context) execute() error {
	t, err := template.NewTemplate(c.src, template.Options{
		Stderr: func() io.Writer { return os.Stderr },
	})
	if err != nil {
		return err
	}
	c.exec = true
	script, err := t.Render(c)
	if err != nil {
		return err
	}
	c.script = script
	log.Debug("running", "script", script)
	if c.run != nil {
		return c.run(script)
	}
	return nil
}
<commit_msg>fix bool parsing from prompt and flag (#463)<commit_after>package local

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/docker/infrakit/pkg/template"
	"github.com/docker/infrakit/pkg/util/exec"
	"github.com/spf13/cobra"
)

const (
	// none is used to determine if the user has set the bool flag value. this allows the
	// use of pipe to a prompt like {{ $foo = flag "flag" "bool" "message" | prompt "foo?" "bool" }}
	none = -1
)

// Context is the context for the running module
type Context struct {
	cmd    *cobra.Command
	src    string
	input  io.Reader
	exec   bool
	run    func(string) error
	script string
}

// name, type, description of the flag, and a default value, which can be nil
// the returned value if the nil value
func (c *Context) defineFlag(name, ftype, desc string, def interface{}) (interface{}, error) {
	switch ftype {

	case "string":
		defaultValue := ""
		if def != nil {
			if v, ok := def.(string); ok {
				defaultValue = v
			} else {
				return nil, fmt.Errorf("default value not a string: %s", name)
			}
		}
		c.cmd.Flags().String(name, defaultValue, desc)
		return defaultValue, nil

	case "int":
		defaultValue := 0 // TODO - encode a nil with a special value?
		if def != nil {
			if v, ok := def.(int); ok {
				defaultValue = v
			} else {
				return nil, fmt.Errorf("default value not an int: %s", name)
			}
		}
		c.cmd.Flags().Int(name, defaultValue, desc)
		return defaultValue, nil

	case "float":
		defaultValue := 0.
		if def != nil {
			if v, ok := def.(float64); ok {
				defaultValue = v
			} else {
				return nil, fmt.Errorf("default value not a float64: %s", name)
			}
		}
		c.cmd.Flags().Float64(name, defaultValue, desc)
		return defaultValue, nil

	case "bool":
		// bool is special in that we want to handle the case of nil --> the flag is not specified
		// so that we can pipe to a prompt if necessary.  pflag does not have the notion of unset
		// flags, so we'd have to hack around it by introducing a string flag if the default is not
		// specified in the template function to define the flag.
		if def != nil {
			// When the default is specified, we cannot use a pipe to prompt.
			// So here just create a bool flag
			defaultValue := false
			switch v := def.(type) {
			case bool:
				defaultValue = v
			case string:
				b, err := strconv.ParseBool(v)
				if err != nil {
					return nil, err
				}
				defaultValue = b
			}
			c.cmd.Flags().Bool(name, defaultValue, desc)
			return defaultValue, nil
		}
		// At definition time, there is no default value, so we use string
		// to model three states: true, false, none
		c.cmd.Flags().Int(name, none, desc)
		return none, nil
	}
	return nil, fmt.Errorf("unknown type %v", ftype)
}

// name, type, description, and default value that can be nil
func (c *Context) getFromFlag(name, ftype, desc string, def interface{}) (interface{}, error) {

	switch ftype {

	case "string":
		return c.cmd.Flags().GetString(name)

	case "int":
		return c.cmd.Flags().GetInt(name)

	case "float":
		return c.cmd.Flags().GetFloat64(name)

	case "bool":
		if def == nil {
			// If default v is not specified, then we assume the flag was defined
			// using a string to handle the none case
			v, err := c.cmd.Flags().GetInt(name)
			if err != nil {
				return none, err
			}
			if v == none {
				return none, nil //
			}
			return v > 0, nil
		}
		return c.cmd.Flags().GetBool(name)
	}

	return nil, nil
}

// returns true if the value v is missing of the type t
func missing(t string, v interface{}) bool {
	if v == nil {
		return true
	}
	switch t {
	case "string":
		return v.(string) == ""
	case "int":
		return v.(int) == 0
	case "float":
		return v.(float64) == 0.
	case "bool":
		return v == none
	}
	return true
}

func parseBool(text string) (bool, error) {
	if b, err := strconv.ParseBool(text); err == nil {
		return b, nil
	}
	switch text {
	case "y", "Y", "yes", "ok", "OK":
		return true, nil
	case "n", "N", "no", "nope":
		return false, nil
	}
	v, err := strconv.Atoi(text)
	return v > 0, err
}

func (c *Context) prompt(prompt, ftype string) (interface{}, error) {
	input := bufio.NewReader(c.input)
	fmt.Fprintf(os.Stderr, "%s ", prompt)
	text, _ := input.ReadString('\n')
	text = strings.Trim(text, " \t\n")
	switch ftype {
	case "string":
		return text, nil
	case "float":
		return strconv.ParseFloat(text, 64)
	case "int":
		if i, err := strconv.Atoi(text); err == nil {
			return i, nil
		}
		// special case -- int can be used to implement a bool if a default is not provided
		// so we need to handle parsing int from text for purpose of determining a bool
		b, err := parseBool(text)
		if err != nil {
			return b, err
		}
		if b {
			return 1, nil
		}
		return 0, nil
	case "bool":
		if b, err := parseBool(text); err == nil {
			return b, nil
		}
		return nil, fmt.Errorf("cannot parse input for boolean: %v", text)
	}
	return nil, nil // don't err, just pass through
}

// Funcs returns the template functions
func (c *Context) Funcs() []template.Function {
	return []template.Function{
		{
			Name: "flag",
			Func: func(n, ftype, desc string, optional ...interface{}) (interface{}, error) {
				if ftype == "" {
					return nil, fmt.Errorf("missing type for variable %v", n)
				}
				var defaultValue interface{}
				if len(optional) > 0 {
					defaultValue = optional[0]
				}
				if c.exec {
					return c.getFromFlag(n, ftype, desc, defaultValue)
				}

				// Defining a flag never returns a printable value that can mess up the template rendering
				return c.defineFlag(n, ftype, desc, defaultValue)
			},
		},
		{
			Name: "prompt",
			Func: func(prompt, ftype string, optional ...interface{}) (interface{}, error) {

				if ftype == "" {
					return nil, fmt.Errorf("missing type for variable prompted")
				}
				if !c.exec {
					return "", nil
				}
				if len(optional) > 0 && !missing(ftype, optional[0]) {
					return optional[0], nil
				}
				return c.prompt(prompt, ftype)
			},
		},
	}
}

// loadBackend determines the backend to use for executing the rendered template text (e.g. run in shell).
// During this phase, the template delimiters are changed to =% %= so put this in the comment {{/* */}}
func (c *Context) loadBackend() error {
	t, err := template.NewTemplate(c.src, template.Options{
		DelimLeft:  "=%",
		DelimRight: "%=",
	})
	if err != nil {
		return err
	}
	t.AddFunc("print",
		func() string {
			c.run = func(script string) error {
				fmt.Println(script)
				return nil
			}
			return ""
		})
	t.AddFunc("sh",
		func(opts ...string) string {
			c.run = func(script string) error {

				cmd := strings.Join(append([]string{"/bin/sh"}, opts...), " ")
				log.Debug("sh", "cmd", cmd)

				return exec.Command(cmd).
					InheritEnvs(true).StartWithStreams(

					exec.Do(exec.SendInput(
						func(stdin io.WriteCloser) error {
							_, err := stdin.Write([]byte(script))
							return err
						})).Then(
						exec.RedirectStdout(os.Stdout)).Then(
						exec.RedirectStderr(os.Stderr),
					).Done(),
				)
			}
			return ""
		})

	_, err = t.Render(c)
	return err
}

// buildFlags from parsing the body which is a template
func (c *Context) buildFlags() error {
	t, err := template.NewTemplate(c.src, template.Options{})
	if err != nil {
		return err
	}
	_, err = t.Render(c)
	return err
}

func (c *Context) execute() error {
	t, err := template.NewTemplate(c.src, template.Options{
		Stderr: func() io.Writer { return os.Stderr },
	})
	if err != nil {
		return err
	}
	c.exec = true
	script, err := t.Render(c)
	if err != nil {
		return err
	}
	c.script = script
	log.Debug("running", "script", script)
	if c.run != nil {
		return c.run(script)
	}
	return nil
}
<|endoftext|>


<commit_before>package filescan

import (
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/OPSWAT/mdcloud-go/pkg/api"
	"github.com/OPSWAT/mdcloud-go/pkg/utils"
	"github.com/fsnotify/fsnotify"
	"github.com/sirupsen/logrus"
)

// ScanOptions from scan cmd
type ScanOptions struct {
	Path         []string
	Headers      []string
	Watcher      bool
	Sanitization bool
	LookupFile   bool
	Poll         bool
}

var watcher *fsnotify.Watcher

// Scan or watches files or path
func Scan(api *api.API, options ScanOptions) {
	if options.Path != nil && len(options.Path) > 0 {
		fname := options.Path[0]
		if !path.IsAbs(fname) {
			wd, err := os.Getwd()
			if err != nil {
				logrus.Panicln(err)
			}
			fname = path.Clean(path.Join(wd, fname))
			options.Path[0] = fname
		}
		fi, err := os.Stat(fname)
		if err != nil {
			logrus.Fatalln(err)
		}
		switch mode := fi.Mode(); {
		case mode.IsDir():
			watchScan(api, options)
		case mode.IsRegular():
			if !options.LookupFile {
				if res, err := api.ScanFile(fname, options.Headers, options.Poll); err == nil {
					logrus.WithField("result", res).Infoln("Scan result")
				} else {
					logrus.Fatalln(err)
				}
			} else {
				lookupSHA1(api, fname, options)
			}
		}
	}
}

func watchScan(api *api.API, options ScanOptions) {
	watcher, _ = fsnotify.NewWatcher()
	defer watcher.Close()

	if err := filepath.Walk(options.Path[0], addPaths); err != nil {
		logrus.Fatalln(err)
	}

	var rate time.Duration
	if api.Type > 0 {
		rate = time.Minute / 10
	} else {
		rate = time.Minute / 100
	}
	throttle := time.Tick(rate)

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if !strings.Contains(event.Name, "/.") && event.Op == fsnotify.Write {
					logrus.WithFields(logrus.Fields{"op": event.Op, "type": event.Name}).Infoln("Change detected")
					<-throttle
					if len(api.Limits) > 0 {
						if res, _ := strconv.Atoi(api.Limits["X-Ratelimit-Remaining"][0]); res > 1 {
							if res < 50 {
								logrus.WithField("X-Ratelimit-Remaining", res).Warnln("limit less than 50")
							}
							go lookupSHA1(api, event.Name, options)
						} else {
							resetIn := api.Limits["X-RateLimit-Reset-In"][0]
							sz := len(resetIn)
							if sz > 0 && resetIn[sz-1] == 's' {
								resetIn = resetIn[:sz-1]
								sleepTime, _ := strconv.Atoi(resetIn)
								logrus.WithField("X-RateLimit-Reset-In", resetIn).Warnln("limit reached, will delay until it is reset")
								time.Sleep(time.Duration(sleepTime) * time.Second)
							}
						}
					} else {
						lookupSHA1(api, event.Name, options)
					}
				}
			case err := <-watcher.Errors:
				logrus.Fatalln(err)
			}
		}
	}()
	<-done
}

func lookupSHA1(api *api.API, filePath string, options ScanOptions) {
	file, err := os.Open(filePath)
	if err != nil {
		logrus.Warningln("Failed to read file")
		return
	}
	stat, err := file.Stat()
	defer file.Close()
	if err == nil {
		if err := filepath.Walk(filePath, addPaths); err != nil {
			logrus.Fatalln(err)
		}
	}
	if !stat.IsDir() {
		resSha1, err := utils.GetFileSHA1(filePath)
		if err == nil {
			if res, err := api.FindOrScan(filePath, resSha1, options.Headers, options.LookupFile, options.Poll); err == nil {
				logrus.Println(res)
			}
		} else {
			logrus.Println(api.ScanFile(filePath, options.Headers, options.Poll))
		}
	}
}

func addPaths(path string, fi os.FileInfo, err error) error {
	if fi.Mode().IsDir() && !strings.Contains(path, "/.") {
		return watcher.Add(path)
	}
	return nil
}
<commit_msg>Update filescan.go<commit_after>package filescan

import (
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/OPSWAT/mdcloud-go/pkg/api"
	"github.com/OPSWAT/mdcloud-go/pkg/utils"
	"github.com/fsnotify/fsnotify"
	"github.com/sirupsen/logrus"
)

// ScanOptions from scan cmd
type ScanOptions struct {
	Path         []string
	Headers      []string
	Watcher      bool
	Sanitization bool
	Unarchive    bool
	LookupFile   bool
	Poll         bool
}

var watcher *fsnotify.Watcher

// Scan or watches files or path
func Scan(api *api.API, options ScanOptions) {
	if options.Path != nil && len(options.Path) > 0 {
		fname := options.Path[0]
		if !path.IsAbs(fname) {
			wd, err := os.Getwd()
			if err != nil {
				logrus.Panicln(err)
			}
			fname = path.Clean(path.Join(wd, fname))
			options.Path[0] = fname
		}
		fi, err := os.Stat(fname)
		if err != nil {
			logrus.Fatalln(err)
		}
		switch mode := fi.Mode(); {
		case mode.IsDir():
			watchScan(api, options)
		case mode.IsRegular():
			if !options.LookupFile {
				if res, err := api.ScanFile(fname, options.Headers, options.Poll); err == nil {
					logrus.WithField("result", res).Infoln("Scan result")
				} else {
					logrus.Fatalln(err)
				}
			} else {
				lookupSHA1(api, fname, options)
			}
		}
	}
}

func watchScan(api *api.API, options ScanOptions) {
	watcher, _ = fsnotify.NewWatcher()
	defer watcher.Close()

	if err := filepath.Walk(options.Path[0], addPaths); err != nil {
		logrus.Fatalln(err)
	}

	var rate time.Duration
	if api.Type > 0 {
		rate = time.Minute / 10
	} else {
		rate = time.Minute / 100
	}
	throttle := time.Tick(rate)

	done := make(chan bool)
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if !strings.Contains(event.Name, "/.") && event.Op == fsnotify.Write {
					logrus.WithFields(logrus.Fields{"op": event.Op, "type": event.Name}).Infoln("Change detected")
					<-throttle
					if len(api.Limits) > 0 {
						if res, _ := strconv.Atoi(api.Limits["X-Ratelimit-Remaining"][0]); res > 1 {
							if res < 50 {
								logrus.WithField("X-Ratelimit-Remaining", res).Warnln("limit less than 50")
							}
							go lookupSHA1(api, event.Name, options)
						} else {
							resetIn := api.Limits["X-RateLimit-Reset-In"][0]
							sz := len(resetIn)
							if sz > 0 && resetIn[sz-1] == 's' {
								resetIn = resetIn[:sz-1]
								sleepTime, _ := strconv.Atoi(resetIn)
								logrus.WithField("X-RateLimit-Reset-In", resetIn).Warnln("limit reached, will delay until it is reset")
								time.Sleep(time.Duration(sleepTime) * time.Second)
							}
						}
					} else {
						lookupSHA1(api, event.Name, options)
					}
				}
			case err := <-watcher.Errors:
				logrus.Fatalln(err)
			}
		}
	}()
	<-done
}

func lookupSHA1(api *api.API, filePath string, options ScanOptions) {
	file, err := os.Open(filePath)
	if err != nil {
		logrus.Warningln("Failed to read file")
		return
	}
	stat, err := file.Stat()
	defer file.Close()
	if err == nil {
		if err := filepath.Walk(filePath, addPaths); err != nil {
			logrus.Fatalln(err)
		}
	}
	if !stat.IsDir() {
		resSha1, err := utils.GetFileSHA1(filePath)
		if err == nil {
			if res, err := api.FindOrScan(filePath, resSha1, options.Headers, options.LookupFile, options.Poll); err == nil {
				logrus.Println(res)
			}
		} else {
			logrus.Println(api.ScanFile(filePath, options.Headers, options.Poll))
		}
	}
}

func addPaths(path string, fi os.FileInfo, err error) error {
	if fi.Mode().IsDir() && !strings.Contains(path, "/.") {
		return watcher.Add(path)
	}
	return nil
}
<|endoftext|>